<commit_before># -*- coding: utf-8 -*-
from __future__ import division, absolute_import, print_function

import sys, os, re

# Check Sphinx version
import sphinx
if sphinx.__version__ < "1.0.1":
    raise RuntimeError("Sphinx 1.0.1 or newer required")

needs_sphinx = '1.0'

# -----------------------------------------------------------------------------
# General configuration
# -----------------------------------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.

sys.path.insert(0, os.path.abspath('../sphinxext'))

extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc',
              'sphinx.ext.intersphinx', 'sphinx.ext.coverage',
              'sphinx.ext.doctest', 'sphinx.ext.autosummary',
              'matplotlib.sphinxext.plot_directive']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# General substitutions.
project = 'NumPy'
copyright = '2008-2009, The Scipy community'

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
import numpy
# The short X.Y version (including .devXXXX, rcX, b1 suffixes if present)
version = re.sub(r'(\d+\.\d+)\.\d+(.*)', r'\1\2', numpy.__version__)
version = re.sub(r'(\.dev\d+).*?$', r'\1', version)
# The full version, including alpha/beta/rc tags.
release = numpy.__version__
print("%s %s" % (version, release))

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# The reST default role (used for this markup: `text`) to use for all documents.
default_role = "autolink"

# List of directories, relative to source directories, that shouldn't be searched
# for source files.
exclude_dirs = []

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# -----------------------------------------------------------------------------
# HTML output
# -----------------------------------------------------------------------------

themedir = os.path.join(os.pardir, 'scipy-sphinx-theme', '_theme')
if not os.path.isdir(themedir):
    raise RuntimeError("Get the scipy-sphinx-theme first, "
                       "via git submodule init && git submodule update")

html_theme = 'scipy'
html_theme_path = [themedir]

if 'scipyorg' in tags:
    # Build for the scipy.org website
    html_theme_options = {
        "edit_link": True,
        "sidebar": "right",
        "scipy_org_logo": True,
        "rootlinks": [("http://scipy.org/", "Scipy.org"),
                      ("http://docs.scipy.org/", "Docs")]
    }
else:
    # Default build
    html_theme_options = {
        "edit_link": False,
        "sidebar": "left",
        "scipy_org_logo": False,
        "rootlinks": []
    }
    html_sidebars = {'index': 'indexsidebar.html'}

html_additional_pages = {
    'index': 'indexcontent.html',
}

html_title = "%s v%s Manual" % (project, version)
html_static_path = ['_static']
html_last_updated_fmt = '%b %d, %Y'

html_use_modindex = True
html_copy_source = False
html_domain_indices = False
html_file_suffix = '.html'

htmlhelp_basename = 'numpy'

pngmath_use_preview = True
pngmath_dvipng_args = ['-gamma', '1.5', '-D', '96', '-bg', 'Transparent']

# -----------------------------------------------------------------------------
# LaTeX output
# -----------------------------------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
_stdauthor = 'Written by the NumPy community'
latex_documents = [
  ('reference/index', 'numpy-ref.tex', 'NumPy Reference',
   _stdauthor, 'manual'),
  ('user/index', 'numpy-user.tex', 'NumPy User Guide',
   _stdauthor, 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
latex_preamble = r'''
\usepackage{amsmath}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}

% In the parameters section, place a newline after the Parameters
% header
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Make Examples/etc section headers smaller and more compact
\makeatletter
\titleformat{\paragraph}{\normalsize\py@HeaderFamily}%
            {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor}
\titlespacing*{\paragraph}{0pt}{1ex}{0pt}
\makeatother

% Fix footer/header
\renewcommand{\chaptermark}[1]{\markboth{\MakeUppercase{\thechapter.\ #1}}{}}
\renewcommand{\sectionmark}[1]{\markright{\MakeUppercase{\thesection.\ #1}}}
'''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_use_modindex = False

# -----------------------------------------------------------------------------
# Texinfo output
# -----------------------------------------------------------------------------

texinfo_documents = [
  ("contents", 'numpy', 'NumPy Documentation', _stdauthor, 'NumPy',
   "NumPy: array processing for numbers, strings, records, and objects.",
   'Programming',
   1),
]

# -----------------------------------------------------------------------------
# Intersphinx configuration
# -----------------------------------------------------------------------------
intersphinx_mapping = {'http://docs.python.org/dev': None}

# -----------------------------------------------------------------------------
# NumPy extensions
# -----------------------------------------------------------------------------

# If we want to do a phantom import from an XML file for all autodocs
phantom_import_file = 'dump.xml'

# Make numpydoc to generate plots for example sections
numpydoc_use_plots = True

# -----------------------------------------------------------------------------
# Autosummary
# -----------------------------------------------------------------------------

import glob
autosummary_generate = glob.glob("reference/*.rst")

# -----------------------------------------------------------------------------
# Coverage checker
# -----------------------------------------------------------------------------
coverage_ignore_modules = r"""
    """.split()
coverage_ignore_functions = r"""
    test($|_) (some|all)true bitwise_not cumproduct pkgload
    generic\.
    """.split()
coverage_ignore_classes = r"""
    """.split()

coverage_c_path = []
coverage_c_regexes = {}
coverage_ignore_c_items = {}

# -----------------------------------------------------------------------------
# Plots
# -----------------------------------------------------------------------------
plot_pre_code = """
import numpy as np
np.random.seed(0)
"""
plot_include_source = True
plot_formats = [('png', 100), 'pdf']

import math
phi = (math.sqrt(5) + 1)/2

plot_rcparams = {
    'font.size': 8,
    'axes.titlesize': 8,
    'axes.labelsize': 8,
    'xtick.labelsize': 8,
    'ytick.labelsize': 8,
    'legend.fontsize': 8,
    'figure.figsize': (3*phi, 3),
    'figure.subplot.bottom': 0.2,
    'figure.subplot.left': 0.2,
    'figure.subplot.right': 0.9,
    'figure.subplot.top': 0.85,
    'figure.subplot.wspace': 0.4,
    'text.usetex': False,
}

# -----------------------------------------------------------------------------
# Source code links
# -----------------------------------------------------------------------------

import inspect
from os.path import relpath, dirname

for name in ['sphinx.ext.linkcode', 'numpydoc.linkcode']:
    try:
        __import__(name)
        extensions.append(name)
        break
    except ImportError:
        pass
else:
    print("NOTE: linkcode extension not found -- no links to source generated")

def linkcode_resolve(domain, info):
    """
    Determine the URL corresponding to Python object
    """
    if domain != 'py':
        return None

    modname = info['module']
    fullname = info['fullname']

    submod = sys.modules.get(modname)
    if submod is None:
        return None

    obj = submod
    for part in fullname.split('.'):
        try:
            obj = getattr(obj, part)
        except:
            return None

    try:
        fn = inspect.getsourcefile(obj)
    except:
        fn = None
    if not fn:
        return None

    try:
        source, lineno = inspect.getsourcelines(obj)
    except:
        lineno = None

    if lineno:
        linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1)
    else:
        linespec = ""

    fn = relpath(fn, start=dirname(numpy.__file__))

    if 'dev' in numpy.__version__:
        return "http://github.com/numpy/numpy/blob/master/numpy/%s%s" % (
           fn, linespec)
    else:
        return "http://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % (
           numpy.__version__, fn, linespec)
<commit_msg>DOC: Add matplotlib inventory for intersphinx.<commit_after># -*- coding: utf-8 -*-
from __future__ import division, absolute_import, print_function

import sys, os, re

# Check Sphinx version
import sphinx
if sphinx.__version__ < "1.0.1":
    raise RuntimeError("Sphinx 1.0.1 or newer required")

needs_sphinx = '1.0'

# -----------------------------------------------------------------------------
# General configuration
# -----------------------------------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.

sys.path.insert(0, os.path.abspath('../sphinxext'))

extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc',
              'sphinx.ext.intersphinx', 'sphinx.ext.coverage',
              'sphinx.ext.doctest', 'sphinx.ext.autosummary',
              'matplotlib.sphinxext.plot_directive']

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# General substitutions.
project = 'NumPy'
copyright = '2008-2009, The Scipy community'

# The default replacements for |version| and |release|, also used in various
# other places throughout the built documents.
#
import numpy
# The short X.Y version (including .devXXXX, rcX, b1 suffixes if present)
version = re.sub(r'(\d+\.\d+)\.\d+(.*)', r'\1\2', numpy.__version__)
version = re.sub(r'(\.dev\d+).*?$', r'\1', version)
# The full version, including alpha/beta/rc tags.
release = numpy.__version__
print("%s %s" % (version, release))

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# The reST default role (used for this markup: `text`) to use for all documents.
default_role = "autolink"

# List of directories, relative to source directories, that shouldn't be searched
# for source files.
exclude_dirs = []

# If true, '()' will be appended to :func: etc. cross-reference text.
add_function_parentheses = False

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# -----------------------------------------------------------------------------
# HTML output
# -----------------------------------------------------------------------------

themedir = os.path.join(os.pardir, 'scipy-sphinx-theme', '_theme')
if not os.path.isdir(themedir):
    raise RuntimeError("Get the scipy-sphinx-theme first, "
                       "via git submodule init && git submodule update")

html_theme = 'scipy'
html_theme_path = [themedir]

if 'scipyorg' in tags:
    # Build for the scipy.org website
    html_theme_options = {
        "edit_link": True,
        "sidebar": "right",
        "scipy_org_logo": True,
        "rootlinks": [("http://scipy.org/", "Scipy.org"),
                      ("http://docs.scipy.org/", "Docs")]
    }
else:
    # Default build
    html_theme_options = {
        "edit_link": False,
        "sidebar": "left",
        "scipy_org_logo": False,
        "rootlinks": []
    }
    html_sidebars = {'index': 'indexsidebar.html'}

html_additional_pages = {
    'index': 'indexcontent.html',
}

html_title = "%s v%s Manual" % (project, version)
html_static_path = ['_static']
html_last_updated_fmt = '%b %d, %Y'

html_use_modindex = True
html_copy_source = False
html_domain_indices = False
html_file_suffix = '.html'

htmlhelp_basename = 'numpy'

pngmath_use_preview = True
pngmath_dvipng_args = ['-gamma', '1.5', '-D', '96', '-bg', 'Transparent']

# -----------------------------------------------------------------------------
# LaTeX output
# -----------------------------------------------------------------------------

# The paper size ('letter' or 'a4').
#latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
#latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, document class [howto/manual]).
_stdauthor = 'Written by the NumPy community'
latex_documents = [
  ('reference/index', 'numpy-ref.tex', 'NumPy Reference',
   _stdauthor, 'manual'),
  ('user/index', 'numpy-user.tex', 'NumPy User Guide',
   _stdauthor, 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
latex_preamble = r'''
\usepackage{amsmath}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}

% In the parameters section, place a newline after the Parameters
% header
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Make Examples/etc section headers smaller and more compact
\makeatletter
\titleformat{\paragraph}{\normalsize\py@HeaderFamily}%
            {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor}
\titlespacing*{\paragraph}{0pt}{1ex}{0pt}
\makeatother

% Fix footer/header
\renewcommand{\chaptermark}[1]{\markboth{\MakeUppercase{\thechapter.\ #1}}{}}
\renewcommand{\sectionmark}[1]{\markright{\MakeUppercase{\thesection.\ #1}}}
'''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_use_modindex = False

# -----------------------------------------------------------------------------
# Texinfo output
# -----------------------------------------------------------------------------

texinfo_documents = [
  ("contents", 'numpy', 'NumPy Documentation', _stdauthor, 'NumPy',
   "NumPy: array processing for numbers, strings, records, and objects.",
   'Programming',
   1),
]

# -----------------------------------------------------------------------------
# Intersphinx configuration
# -----------------------------------------------------------------------------
intersphinx_mapping = {
    'python': ('https://docs.python.org/dev', None),
    'matplotlib': ('http://matplotlib.org', None)
}

# -----------------------------------------------------------------------------
# NumPy extensions
# -----------------------------------------------------------------------------

# If we want to do a phantom import from an XML file for all autodocs
phantom_import_file = 'dump.xml'

# Make numpydoc to generate plots for example sections
numpydoc_use_plots = True

# -----------------------------------------------------------------------------
# Autosummary
# -----------------------------------------------------------------------------

import glob
autosummary_generate = glob.glob("reference/*.rst")

# -----------------------------------------------------------------------------
# Coverage checker
# -----------------------------------------------------------------------------
coverage_ignore_modules = r"""
    """.split()
coverage_ignore_functions = r"""
    test($|_) (some|all)true bitwise_not cumproduct pkgload
    generic\.
    """.split()
coverage_ignore_classes = r"""
    """.split()

coverage_c_path = []
coverage_c_regexes = {}
coverage_ignore_c_items = {}

# -----------------------------------------------------------------------------
# Plots
# -----------------------------------------------------------------------------
plot_pre_code = """
import numpy as np
np.random.seed(0)
"""
plot_include_source = True
plot_formats = [('png', 100), 'pdf']

import math
phi = (math.sqrt(5) + 1)/2

plot_rcparams = {
    'font.size': 8,
    'axes.titlesize': 8,
    'axes.labelsize': 8,
    'xtick.labelsize': 8,
    'ytick.labelsize': 8,
    'legend.fontsize': 8,
    'figure.figsize': (3*phi, 3),
    'figure.subplot.bottom': 0.2,
    'figure.subplot.left': 0.2,
    'figure.subplot.right': 0.9,
    'figure.subplot.top': 0.85,
    'figure.subplot.wspace': 0.4,
    'text.usetex': False,
}

# -----------------------------------------------------------------------------
# Source code links
# -----------------------------------------------------------------------------

import inspect
from os.path import relpath, dirname

for name in ['sphinx.ext.linkcode', 'numpydoc.linkcode']:
    try:
        __import__(name)
        extensions.append(name)
        break
    except ImportError:
        pass
else:
    print("NOTE: linkcode extension not found -- no links to source generated")

def linkcode_resolve(domain, info):
    """
    Determine the URL corresponding to Python object
    """
    if domain != 'py':
        return None

    modname = info['module']
    fullname = info['fullname']

    submod = sys.modules.get(modname)
    if submod is None:
        return None

    obj = submod
    for part in fullname.split('.'):
        try:
            obj = getattr(obj, part)
        except:
            return None

    try:
        fn = inspect.getsourcefile(obj)
    except:
        fn = None
    if not fn:
        return None

    try:
        source, lineno = inspect.getsourcelines(obj)
    except:
        lineno = None

    if lineno:
        linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1)
    else:
        linespec = ""

    fn = relpath(fn, start=dirname(numpy.__file__))

    if 'dev' in numpy.__version__:
        return "http://github.com/numpy/numpy/blob/master/numpy/%s%s" % (
           fn, linespec)
    else:
        return "http://github.com/numpy/numpy/blob/v%s/numpy/%s%s" % (
           numpy.__version__, fn, linespec)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# skimage documentation build configuration file, created by
# sphinx-quickstart on Sat Aug 22 13:00:30 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))

curpath = os.path.dirname(__file__)
sys.path.append(os.path.join(curpath, '..', 'ext'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc',
              'sphinx.ext.autosummary', 'plot2rst',
              'sphinx.ext.intersphinx']

# Determine if the matplotlib has a recent enough version of the
# plot_directive, otherwise use the local fork.
try:
    from matplotlib.sphinxext import plot_directive
except ImportError:
    use_matplotlib_plot_directive = False
else:
    try:
        use_matplotlib_plot_directive = (plot_directive.__version__ >= 2)
    except AttributeError:
        use_matplotlib_plot_directive = False

if use_matplotlib_plot_directive:
    extensions.append('matplotlib.sphinxext.plot_directive')
else:
    extensions.append('plot_directive')

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.txt'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'skimage'
copyright = '2013, the scikit-image team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.

setup_lines = open('../../setup.py').readlines()
version = 'vUndefined'
for l in setup_lines:
    if l.startswith('VERSION'):
        version = l.split("'")[1]
        break

# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'scikit-image'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['themes']

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = 'skimage v%s docs' % version

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
   '**': ['navigation.html',
          'localtoc.html',
          'versions.html'],
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'scikitimagedoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
latex_paper_size = 'a4'

# The font size ('10pt', '11pt' or '12pt').
latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('contents', 'scikit-image.tex', u'The scikit-image Documentation',
   u'scikit-image development team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
latex_preamble = r'''
\usepackage{enumitem}
\setlistdepth{100}

\usepackage{amsmath}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}

% In the parameters section, place a newline after the Parameters header
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Make Examples/etc section headers smaller and more compact
\makeatletter
\titleformat{\paragraph}{\normalsize\py@HeaderFamily}%
            {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor}
\titlespacing*{\paragraph}{0pt}{1ex}{0pt}
\makeatother

'''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_use_modindex = True

# -----------------------------------------------------------------------------
# Numpy extensions
# -----------------------------------------------------------------------------
numpydoc_show_class_members = False

# -----------------------------------------------------------------------------
# Plots
# -----------------------------------------------------------------------------
plot_basedir = os.path.join(curpath, "plots")
plot_pre_code = """
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(0)

import matplotlib
matplotlib.rcParams.update({
    'font.size': 14,
    'axes.titlesize': 12,
    'axes.labelsize': 10,
    'xtick.labelsize': 8,
    'ytick.labelsize': 8,
    'legend.fontsize': 10,
    'figure.subplot.bottom': 0.2,
    'figure.subplot.left': 0.2,
    'figure.subplot.right': 0.9,
    'figure.subplot.top': 0.85,
    'figure.subplot.wspace': 0.4,
    'text.usetex': False,
})

"""
plot_include_source = True
plot_formats = [('png', 100), ('pdf', 100)]

plot2rst_index_name = 'README'
plot2rst_rcparams = {'image.cmap' : 'gray',
                     'image.interpolation' : 'none'}

# -----------------------------------------------------------------------------
# intersphinx
# -----------------------------------------------------------------------------
_python_doc_base = 'http://docs.python.org/2.7'
intersphinx_mapping = {
    _python_doc_base: None,
    'http://docs.scipy.org/doc/numpy': None,
    'http://docs.scipy.org/doc/scipy/reference': None,
    'http://scikit-learn.org/stable': None
}
<commit_msg>Avoid duplicate python module index<commit_after># -*- coding: utf-8 -*-
#
# skimage documentation build configuration file, created by
# sphinx-quickstart on Sat Aug 22 13:00:30 2009.
#
# This file is execfile()d with the current directory set to its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys, os

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.append(os.path.abspath('.'))

curpath = os.path.dirname(__file__)
sys.path.append(os.path.join(curpath, '..', 'ext'))

# -- General configuration -----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be extensions
# coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
extensions = ['sphinx.ext.autodoc', 'sphinx.ext.pngmath', 'numpydoc',
              'sphinx.ext.autosummary', 'plot2rst',
              'sphinx.ext.intersphinx']

# Determine if the matplotlib has a recent enough version of the
# plot_directive, otherwise use the local fork.
try:
    from matplotlib.sphinxext import plot_directive
except ImportError:
    use_matplotlib_plot_directive = False
else:
    try:
        use_matplotlib_plot_directive = (plot_directive.__version__ >= 2)
    except AttributeError:
        use_matplotlib_plot_directive = False

if use_matplotlib_plot_directive:
    extensions.append('matplotlib.sphinxext.plot_directive')
else:
    extensions.append('plot_directive')

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.txt'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'skimage'
copyright = '2013, the scikit-image team'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.

setup_lines = open('../../setup.py').readlines()
version = 'vUndefined'
for l in setup_lines:
    if l.startswith('VERSION'):
        version = l.split("'")[1]
        break

# The full version, including alpha/beta/rc tags.
release = version

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of documents that shouldn't be included in the build.
#unused_docs = []

# List of directories, relative to source directory, that shouldn't be searched
# for source files.
exclude_trees = []

# The reST default role (used for this markup: `text`) to use for all documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# -- Options for HTML output ---------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
html_theme = 'scikit-image'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
html_theme_path = ['themes']

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
html_title = 'skimage v%s docs' % version

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
html_sidebars = {
   '**': ['navigation.html',
          'localtoc.html',
          'versions.html'],
}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
#html_use_modindex = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'scikitimagedoc'

# -- Options for LaTeX output --------------------------------------------------

# The paper size ('letter' or 'a4').
latex_paper_size = 'a4'

# The font size ('10pt', '11pt' or '12pt').
latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass [howto/manual]).
latex_documents = [
  ('contents', 'scikit-image.tex', u'The scikit-image Documentation',
   u'scikit-image development team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
#latex_use_parts = False

# Additional stuff for the LaTeX preamble.
latex_preamble = r'''
\usepackage{enumitem}
\setlistdepth{100}

\usepackage{amsmath}
\DeclareUnicodeCharacter{00A0}{\nobreakspace}

% In the parameters section, place a newline after the Parameters header
\usepackage{expdlist}
\let\latexdescription=\description
\def\description{\latexdescription{}{} \breaklabel}

% Make Examples/etc section headers smaller and more compact
\makeatletter
\titleformat{\paragraph}{\normalsize\py@HeaderFamily}%
            {\py@TitleColor}{0em}{\py@TitleColor}{\py@NormalColor}
\titlespacing*{\paragraph}{0pt}{1ex}{0pt}
\makeatother

'''

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
latex_use_modindex = False

# -----------------------------------------------------------------------------
# Numpy extensions
# -----------------------------------------------------------------------------
numpydoc_show_class_members = False

# -----------------------------------------------------------------------------
# Plots
# -----------------------------------------------------------------------------
plot_basedir = os.path.join(curpath, "plots")
plot_pre_code = """
import numpy as np
import matplotlib.pyplot as plt
np.random.seed(0)

import matplotlib
matplotlib.rcParams.update({
    'font.size': 14,
    'axes.titlesize': 12,
    'axes.labelsize': 10,
    'xtick.labelsize': 8,
    'ytick.labelsize': 8,
    'legend.fontsize': 10,
    'figure.subplot.bottom': 0.2,
    'figure.subplot.left': 0.2,
    'figure.subplot.right': 0.9,
    'figure.subplot.top': 0.85,
    'figure.subplot.wspace': 0.4,
    'text.usetex': False,
})

"""
plot_include_source = True
plot_formats = [('png', 100), ('pdf', 100)]

plot2rst_index_name = 'README'
plot2rst_rcparams = {'image.cmap' : 'gray',
                     'image.interpolation' : 'none'}

# -----------------------------------------------------------------------------
# intersphinx
# -----------------------------------------------------------------------------
_python_doc_base = 'http://docs.python.org/2.7'
intersphinx_mapping = {
    _python_doc_base: None,
    'http://docs.scipy.org/doc/numpy': None,
    'http://docs.scipy.org/doc/scipy/reference': None,
    'http://scikit-learn.org/stable': None
}
<|endoftext|>


<commit_before>import scipy.optimize as opt
import numpy as np

class ModelBase(object):
    def __init__(self,x0=None,bounds=None):
        self.x0 = x0
        self.bounds = bounds

    def parameter_optimization(self,average_daily_usages,observed_daily_temps):
        """Returns parameters which, according to an optimization routine in
        `scipy.optimize`, minimize the sum of squared errors between observed
        usages and the output of the a model which takes observed_daily_temps
        and returns usage estimates.
        """
        # ignore nans
        average_daily_usages = np.ma.masked_array(average_daily_usages,np.isnan(average_daily_usages))

        # precalculate temps
        n_daily_temps = np.array([len(temps) for temps in observed_daily_temps])

        def objective_function(params):
            usages_est = self.compute_usage_estimates(params,observed_daily_temps)
            avg_usages_est = usages_est/n_daily_temps
            return np.sum((average_daily_usages - avg_usages_est)**2)

        assert len(average_daily_usages) == len(observed_daily_temps)

        result = opt.minimize(objective_function,x0=self.x0,bounds=self.bounds)
        params = result.x
        return params

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Return usage estimates given model parameters and temperature
        observations as an iterable of iterables containing daily temperatures
        (e.g. [[70.1,71.7,46.3],[10.1,15.1,16.0]]). Content and format of
        `params` must be determined by inheriting classes. Function must be
        implemented by all inheriting classes.
        """
        raise NotImplementedError

class HDDCDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a combined, dual balance point,
        heating/cooling degree day model. Parameters are given in the form
        `params = (ts_low, ts_high, base_load, bp_low, bp_diff)`, in which:

        - `ts_low` is the (generally positive) temperature sensitivity
          (units: usage per hdd) beyond the lower (heating degree day)
          balance point
        - `ts_high` is the (generally positive) temperature sensitivity
          (units: usage per cdd) beyond the upper (cooling degree day balance
          point
        - `base_load` is the daily non-temperature-related usage
        - `bp_low` is the reference temperature of the lower (hdd) balance
          point
        - bd_diff is the (generally positive) difference between the
          implicitly defined `bp_high` and `bp_low`

        """
        # get parameters
        ts_low,ts_high,base_load,bp_low,bp_diff = params
        bp_high = bp_low + bp_diff

        hdds = []
        cdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            hdd = 0
            cdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp <= bp_low:
                        hdd += bp_low - daily_temp
                    elif daily_temp >= bp_high:
                        cdd += daily_temp - bp_high
                bl += base_load
            hdds.append(hdd)
            cdds.append(cdd)
            base_loads.append(bl)

        estimates = np.array([bl + ts_low*hdd + ts_high*cdd
                              for hdd,cdd,bl in zip(hdds,cdds,base_loads)])

        return estimates

class HDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a simple single balance point, heating
        degree day model. Parameters are given in the form
        `params = (reference_temperature, base_level_consumption,
        heating_slope)`, in which:

        - `reference_temperature` is the temperature base of the heating
          degree day balance point
        - `base_level_consumption` is the daily non-temperature-related
          usage
        - `heating_slope` is the (generally positive) temperature
          sensitivity (units: usage per hdd) beyond the heating degree day
          reference temperature

        """
        # get parameters
        reference_temperature,base_level_consumption,heating_slope = params

        hdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            hdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp <= reference_temperature:
                        hdd += reference_temperature - daily_temp
                bl += base_level_consumption
            hdds.append(hdd)
            base_loads.append(bl)

        estimates = np.array([bl + heating_slope*hdd
                              for hdd,bl in zip(hdds,base_loads)])

        return estimates

class CDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a simple single balance point, heating
        degree day model. Parameters are given in the form
        `params = (reference_temperature, base_level_consumption,
        cooling_slope)`, in which:

        - `reference_temperature` is the temperature base of the cooling
          degree day balance point
        - `base_level_consumption` is the daily non-temperature-related
          usage
        - `cooling_slope` is the (generally positive) temperature
          sensitivity (units: usage per cdd) beyond the cooling degree day
          reference temperature

        """
        # get parameters
        reference_temperature,base_level_consumption,cooling_slope = params

        cdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            cdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp >= reference_temperature:
                        cdd += daily_temp - reference_temperature
                bl += base_level_consumption
            cdds.append(cdd)
            base_loads.append(bl)

        estimates = np.array([bl + cooling_slope*cdd
                              for cdd,bl in zip(cdds,base_loads)])

        return estimates
<commit_msg>update docstring<commit_after>import scipy.optimize as opt
import numpy as np

class ModelBase(object):
    def __init__(self,x0=None,bounds=None):
        self.x0 = x0
        self.bounds = bounds

    def parameter_optimization(self,average_daily_usages,observed_daily_temps):
        """Returns parameters which, according to an optimization routine in
        `scipy.optimize`, minimize the sum of squared errors between observed
        usages and the output of the a model which takes observed_daily_temps
        and returns usage estimates.
        """
        # ignore nans
        average_daily_usages = np.ma.masked_array(average_daily_usages,np.isnan(average_daily_usages))

        # precalculate temps
        n_daily_temps = np.array([len(temps) for temps in observed_daily_temps])

        def objective_function(params):
            usages_est = self.compute_usage_estimates(params,observed_daily_temps)
            avg_usages_est = usages_est/n_daily_temps
            return np.sum((average_daily_usages - avg_usages_est)**2)

        assert len(average_daily_usages) == len(observed_daily_temps)

        result = opt.minimize(objective_function,x0=self.x0,bounds=self.bounds)
        params = result.x
        return params

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Return usage estimates given model parameters and temperature
        observations as an iterable of iterables containing daily temperatures
        (e.g. [[70.1,71.7,46.3],[10.1,15.1,16.0]]). Content and format of
        `params` must be determined by inheriting classes. Function must be
        implemented by all inheriting classes.
        """
        raise NotImplementedError

class HDDCDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a combined, dual balance point,
        heating/cooling degree day model. Parameters are given in the form
        `params = (ts_low, ts_high, base_load, bp_low, bp_diff)`, in which:

        - `ts_low` is the (generally positive) temperature sensitivity
          (units: usage per hdd) beyond the lower (heating degree day)
          balance point
        - `ts_high` is the (generally positive) temperature sensitivity
          (units: usage per cdd) beyond the upper (cooling degree day balance
          point
        - `base_load` is the daily non-temperature-related usage
        - `bp_low` is the reference temperature of the lower (hdd) balance
          point
        - `bd_diff` is the (generally positive) difference between the
          implicitly defined `bp_high` and `bp_low`

        """
        # get parameters
        ts_low,ts_high,base_load,bp_low,bp_diff = params
        bp_high = bp_low + bp_diff

        hdds = []
        cdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            hdd = 0
            cdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp <= bp_low:
                        hdd += bp_low - daily_temp
                    elif daily_temp >= bp_high:
                        cdd += daily_temp - bp_high
                bl += base_load
            hdds.append(hdd)
            cdds.append(cdd)
            base_loads.append(bl)

        estimates = np.array([bl + ts_low*hdd + ts_high*cdd
                              for hdd,cdd,bl in zip(hdds,cdds,base_loads)])

        return estimates

class HDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a simple single balance point, heating
        degree day model. Parameters are given in the form
        `params = (reference_temperature, base_level_consumption,
        heating_slope)`, in which:

        - `reference_temperature` is the temperature base of the heating
          degree day balance point
        - `base_level_consumption` is the daily non-temperature-related
          usage
        - `heating_slope` is the (generally positive) temperature
          sensitivity (units: usage per hdd) beyond the heating degree day
          reference temperature

        """
        # get parameters
        reference_temperature,base_level_consumption,heating_slope = params

        hdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            hdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp <= reference_temperature:
                        hdd += reference_temperature - daily_temp
                bl += base_level_consumption
            hdds.append(hdd)
            base_loads.append(bl)

        estimates = np.array([bl + heating_slope*hdd
                              for hdd,bl in zip(hdds,base_loads)])

        return estimates

class CDDBalancePointModel(ModelBase):

    @staticmethod
    def compute_usage_estimates(params,observed_daily_temps):
        """Returns usage estimates for a simple single balance point, heating
        degree day model. Parameters are given in the form
        `params = (reference_temperature, base_level_consumption,
        cooling_slope)`, in which:

        - `reference_temperature` is the temperature base of the cooling
          degree day balance point
        - `base_level_consumption` is the daily non-temperature-related
          usage
        - `cooling_slope` is the (generally positive) temperature
          sensitivity (units: usage per cdd) beyond the cooling degree day
          reference temperature

        """
        # get parameters
        reference_temperature,base_level_consumption,cooling_slope = params

        cdds = []
        base_loads = []
        for interval_daily_temps in observed_daily_temps:
            cdd = 0
            bl = 0
            for daily_temp in interval_daily_temps:
                if not np.isnan(daily_temp):
                    if daily_temp >= reference_temperature:
                        cdd += daily_temp - reference_temperature
                bl += base_level_consumption
            cdds.append(cdd)
            base_loads.append(bl)

        estimates = np.array([bl + cooling_slope*cdd
                              for cdd,bl in zip(cdds,base_loads)])

        return estimates
<|endoftext|>


<commit_before>#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from hydra_agent.log import agent_log
from hydra_agent import shell

import os
import glob
import re

def _dev_major_minor(path):
    """Return a string if 'path' is a block device or link to one, else return None"""
    from stat import S_ISBLK
    try:
        s = os.stat(path)
    except OSError:
        return None

    if S_ISBLK(s.st_mode):
        return "%d:%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))
    else:
        return None

def _find_block_devs(folder):
    # Map of major_minor to path
    result = {}
    for path in glob.glob(os.path.join(folder, "*")):
        mm = _dev_major_minor(path)
        if mm:
            result[mm] = path

    return result

def _get_vgs():
    out = shell.try_run(["vgs", "--units", "b", "--noheadings", "-o", "vg_name,vg_uuid,vg_size"])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size)

def _get_lvs(vg_name):
    out = shell.try_run(["lvs", "--units", "b", "--noheadings", "-o", "lv_name,lv_uuid,lv_size,lv_path", vg_name])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str, path = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size, path)

old_udev = None
def _device_node(device_name, major_minor, path, size, parent):
    # This is a WTF, when I declare old_udev at module scope this function can't see it?!?!
    global old_udev
    # Old (RHEL5) version of scsi_id has a different command line
    # syntax to new (RHEL6) version.  Tell them apart with --version
    # (old one doesn't have --version, new one does)
    SCSI_ID_PATH = "/sbin/scsi_id"
    if old_udev == None:
        rc, out, err = shell.run(['scsi_id', '--version'])
        old_udev = (rc != 0)

    # Note: we use -p 0x80 with scsi_id in order to get
    # a textual id along the lines of:
    # * SQEMU    QEMU HARDDISK  WD-deadbeef0
    # * SOPNFILERVIRTUAL-DISK   9iI2mH-4Ddf-k3Ov
    # * SDDN     S2A 9550       058C4A531100
    # As well as being more readable than the 0x83 ID, this
    # gets us the serial number for QEMU devices if the user
    # has set one, whereas that doesn't show up in 0x83.

    if old_udev:
        # Old scsi_id, operates on a /sys reference
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", "-s", "/block/%s" % device_name])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()
    else:
        # New scsi_id, always operates directly on a device
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", path])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()

    # The downside to using -p 0x80 is that if the user hasn't manually
    # set serials for their scsi devices, multiple different devices on
    # the same host return the same string, so we need an explicit
    # exclusion for that
    if serial == "SQEMU    QEMU HARDDISK  0":
        serial = None

    return {'major_minor': major_minor,
            'path': path,
            'serial': serial,
            'size': size,
            'parent': parent}

def _parse_sys_block():
    mapper_devs = _find_block_devs("/dev/mapper/")
    by_id_devs = _find_block_devs("/dev/disk/by-id/")

    def get_path(major_minor, device_name):
        # Try to find device nodes for these:
        fallback_dev_path = os.path.join("/dev/", device_name)
        # * First look in /dev/mapper
        if major_minor in mapper_devs:
            return mapper_devs[major_minor]
        # * Then try /dev/disk/by-id
        elif major_minor in by_id_devs:
            return by_id_devs[major_minor]
        # * Then fall back to just /dev
        elif os.path.exists(fallback_dev_path):
            return fallback_dev_path
        else:
            agent_log.warning("Could not find device node for %s (%s)" % (major_minor, fallback_dev_path))
            return None

    block_device_nodes = {}
    node_block_devices = {}

    def parse_block_dir(dev_dir, parent = None):
        """Parse a dir like /sys/block/sda (must contain 'dev' and 'size')"""
        device_name = dev_dir.split(os.sep)[-1]
        major_minor = open(os.path.join(dev_dir, "dev")).read().strip()
        size = int(open(os.path.join(dev_dir, "size")).read().strip()) * 512

        # Exclude zero-sized devices
        if size == 0:
            return

        # Exclude ramdisks and floppy drives
        if re.search("^ram\d+$", device_name) or \
           re.search("^fd\d+$", device_name) or \
           re.search("^sr\d+$", device_name):
            return

        # TODO: more general check for read-only devices in addition to 'sr\d' check above

        # Resolve a major:minor to a /dev/foo
        path = get_path(major_minor, device_name)
        if path:
            block_device_nodes[major_minor] = _device_node(device_name, major_minor, path, size, parent)
            node_block_devices[path] = major_minor

        return major_minor

    for dev_dir in glob.glob("/sys/block/*"):
        major_minor = parse_block_dir(dev_dir)

        partitions = glob.glob(os.path.join(dev_dir, "*/dev"))
        for p in partitions:
            parse_block_dir(os.path.split(p)[0], parent = major_minor)

    return block_device_nodes, node_block_devices

def device_scan(args):
    # Map of block devices major:minors to /dev/ path.
    block_device_nodes, node_block_devices = _parse_sys_block()

    # XXX
    # because it's a pain, and rare, we're not handling "partitions in LVs".
    # normal partitions are fine because they appear as subdirs of the device
    # in /sys/block, but for some reason partitions in LVs don't do that.

    vgs = {}
    lvs = {}
    for vg_name, vg_uuid, vg_size in _get_vgs():
        vgs[vg_name] = {
                'name': vg_name,
                'uuid': vg_uuid,
                'size': vg_size,
                'pvs_major_minor': []}
        lvs[vg_name] = {}
        for lv_name, lv_uuid, lv_size, lv_path in _get_lvs(vg_name):
            lvs[vg_name][lv_name] = {
                'name': lv_name,
                'uuid': lv_uuid,
                'size': lv_size}

    # Map multipath name to set of devices
    mpaths = {}

    stdout = shell.try_run(['dmsetup', 'table'])
    if stdout.strip() == "No devices found":
        dm_lines = []
    else:
        dm_lines = [i for i in stdout.split("\n") if len(i) > 0]
    for line in dm_lines:
        tokens = line.split()
        name = tokens[0].strip(":")
        num_sectors = int(tokens[2])
        dm_type = tokens[3]
        node_class = None

        node_path = os.path.join("/dev/mapper", name)
        block_device = node_block_devices[node_path]

        if dm_type in ['linear', 'striped']:
            obj_class = 'lv'
            # This is an LVM LV
            if dm_type == 'striped':
                # List of striped devices
                dev_indices = range(6, len(tokens), 2)
                devices = [tokens[i] for i in dev_indices]
            elif dm_type == 'linear':
                # Single device linear range
                devices = [tokens[4]]

            # When a name has a "-" in it, DM prints a double hyphen in the output
            # So for an LV called "my-lv" you get VolGroup00-my--lv
            vg_name, lv_name = re.search("(.*[^-])-([^-].*)", name).groups()
            vg_name = vg_name.replace("--", "-")
            lv_name = lv_name.replace("--", "-")

            lvs[vg_name][lv_name]['block_device'] = block_device

            devices = [block_device_nodes[i]['major_minor'] for i in devices]
            vgs[vg_name]['pvs_major_minor'] = list(set(vgs[vg_name]['pvs_major_minor']) | set(devices))
        elif dm_type == 'multipath':
            # This is a multipath device, there will be a list of devices like this:
            # "round-robin 0 1 1 8:80 1000"
            dev_indices = range(12, len(tokens), 6)
            devices = [tokens[i] for i in dev_indices]
            devices = [block_device_nodes[i] for i in devices]
            if name in mpaths:
                raise RuntimeError("Duplicated mpath device %s" % name)

            mpaths[name] = {
                    "name": name,
                    "block_device": block_device,
                    "nodes": devices
                    }
        else:
            continue

    # Anything in fstab or that is mounted
    # TODO: move these out somewhere
    from hydra_agent.legacy_audit import Fstab, Mounts
    fstab = Fstab()
    mounts = Mounts()
    from itertools import chain
    bdev_to_local_fs = {}
    for dev, mntpnt, fstype in chain(fstab.all(), mounts.all()):
        mm = _dev_major_minor(dev)
        if mm and fstype != 'lustre' and mm in block_device_nodes:
            bdev_to_local_fs[mm] = (mntpnt, fstype)

    return {"vgs": vgs, "lvs": lvs, "mpath": mpaths, "devs": block_device_nodes, "local_fs": bdev_to_local_fs}
<commit_msg>* Check for read-only or removed devices * Handle partitions in LVs (up to a single level, at least)<commit_after>#
# ==============================
# Copyright 2011 Whamcloud, Inc.
# ==============================

from hydra_agent.log import agent_log
from hydra_agent import shell

import os
import glob
import re

def _dev_major_minor(path):
    """Return a string if 'path' is a block device or link to one, else return None"""
    from stat import S_ISBLK
    try:
        s = os.stat(path)
    except OSError:
        return None

    if S_ISBLK(s.st_mode):
        return "%d:%d" % (os.major(s.st_rdev), os.minor(s.st_rdev))
    else:
        return None

def _find_block_devs(folder):
    # Map of major_minor to path
    result = {}
    for path in glob.glob(os.path.join(folder, "*")):
        mm = _dev_major_minor(path)
        if mm:
            result[mm] = path

    return result

def _get_vgs():
    out = shell.try_run(["vgs", "--units", "b", "--noheadings", "-o", "vg_name,vg_uuid,vg_size"])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size)

def _get_lvs(vg_name):
    out = shell.try_run(["lvs", "--units", "b", "--noheadings", "-o", "lv_name,lv_uuid,lv_size,lv_path", vg_name])

    lines = [l for l in out.split("\n") if len(l) > 0]
    for line in lines:
        name, uuid, size_str, path = line.split()
        size = int(size_str[0:-1], 10)
        yield (name, uuid, size, path)

old_udev = None
def _device_node(device_name, major_minor, path, size, parent):
    # This is a WTF, when I declare old_udev at module scope this function can't see it?!?!
    global old_udev
    # Old (RHEL5) version of scsi_id has a different command line
    # syntax to new (RHEL6) version.  Tell them apart with --version
    # (old one doesn't have --version, new one does)
    SCSI_ID_PATH = "/sbin/scsi_id"
    if old_udev == None:
        rc, out, err = shell.run(['scsi_id', '--version'])
        old_udev = (rc != 0)

    # Note: we use -p 0x80 with scsi_id in order to get
    # a textual id along the lines of:
    # * SQEMU    QEMU HARDDISK  WD-deadbeef0
    # * SOPNFILERVIRTUAL-DISK   9iI2mH-4Ddf-k3Ov
    # * SDDN     S2A 9550       058C4A531100
    # As well as being more readable than the 0x83 ID, this
    # gets us the serial number for QEMU devices if the user
    # has set one, whereas that doesn't show up in 0x83.

    if old_udev:
        # Old scsi_id, operates on a /sys reference
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", "-s", "/block/%s" % device_name])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()
    else:
        # New scsi_id, always operates directly on a device
        rc, out, err = shell.run(["scsi_id", "-g", "-p", "0x80", path])
        if rc != 0:
            serial = None
        else:
            serial = out.strip()

    # The downside to using -p 0x80 is that if the user hasn't manually
    # set serials for their scsi devices, multiple different devices on
    # the same host return the same string, so we need an explicit
    # exclusion for that
    if serial == "SQEMU    QEMU HARDDISK  0":
        serial = None

    return {'major_minor': major_minor,
            'path': path,
            'serial': serial,
            'size': size,
            'parent': parent}

def _parse_sys_block():
    mapper_devs = _find_block_devs("/dev/mapper/")
    by_id_devs = _find_block_devs("/dev/disk/by-id/")

    def get_path(major_minor, device_name):
        # Try to find device nodes for these:
        fallback_dev_path = os.path.join("/dev/", device_name)
        # * First look in /dev/mapper
        if major_minor in mapper_devs:
            return mapper_devs[major_minor]
        # * Then try /dev/disk/by-id
        elif major_minor in by_id_devs:
            return by_id_devs[major_minor]
        # * Then fall back to just /dev
        elif os.path.exists(fallback_dev_path):
            return fallback_dev_path
        else:
            agent_log.warning("Could not find device node for %s (%s)" % (major_minor, fallback_dev_path))
            return None

    block_device_nodes = {}
    node_block_devices = {}

    def parse_block_dir(dev_dir, parent = None):
        """Parse a dir like /sys/block/sda (must contain 'dev' and 'size')"""
        device_name = dev_dir.split(os.sep)[-1]
        major_minor = open(os.path.join(dev_dir, "dev")).read().strip()
        size = int(open(os.path.join(dev_dir, "size")).read().strip()) * 512

        # Exclude zero-sized devices
        if size == 0:
            return

        # Exclude ramdisks, floppy drives, obvious cdroms
        if re.search("^ram\d+$", device_name) or \
           re.search("^fd\d+$", device_name) or \
           re.search("^sr\d+$", device_name):
            return

        # Exclude read-only or removed devices
        try:
            open("/dev/%s" % device_name, 'w')
        except IOError, e:
            import errno
            # Python errno doesn't include this code
            NO_MEDIA_ERRNO = 123
            if e.errno == errno.EROFS or e.errno == NO_MEDIA_ERRNO:
                return

        # Resolve a major:minor to a /dev/foo
        path = get_path(major_minor, device_name)
        if path:
            block_device_nodes[major_minor] = _device_node(device_name, major_minor, path, size, parent)
            node_block_devices[path] = major_minor

        return major_minor

    for dev_dir in glob.glob("/sys/block/*"):
        major_minor = parse_block_dir(dev_dir)

        partitions = glob.glob(os.path.join(dev_dir, "*/dev"))
        for p in partitions:
            parse_block_dir(os.path.split(p)[0], parent = major_minor)

    return block_device_nodes, node_block_devices

def device_scan(args):
    # Map of block devices major:minors to /dev/ path.
    block_device_nodes, node_block_devices = _parse_sys_block()

    vgs = {}
    lvs = {}
    for vg_name, vg_uuid, vg_size in _get_vgs():
        vgs[vg_name] = {
                'name': vg_name,
                'uuid': vg_uuid,
                'size': vg_size,
                'pvs_major_minor': []}
        lvs[vg_name] = {}
        for lv_name, lv_uuid, lv_size, lv_path in _get_lvs(vg_name):
            lvs[vg_name][lv_name] = {
                'name': lv_name,
                'uuid': lv_uuid,
                'size': lv_size}

    # Map multipath name to set of devices
    mpaths = {}

    stdout = shell.try_run(['dmsetup', 'table'])
    if stdout.strip() == "No devices found":
        dm_lines = []
    else:
        dm_lines = [i for i in stdout.split("\n") if len(i) > 0]
    for line in dm_lines:
        tokens = line.split()
        name = tokens[0].strip(":")
        num_sectors = int(tokens[2])
        dm_type = tokens[3]
        node_class = None

        node_path = os.path.join("/dev/mapper", name)
        block_device = node_block_devices[node_path]

        if dm_type in ['linear', 'striped']:
            obj_class = 'lv'
            # This is an LVM LV
            if dm_type == 'striped':
                # List of striped devices
                dev_indices = range(6, len(tokens), 2)
                devices = [tokens[i] for i in dev_indices]
            elif dm_type == 'linear':
                # Single device linear range
                devices = [tokens[4]]

            # When a name has a "-" in it, DM prints a double hyphen in the output
            # So for an LV called "my-lv" you get VolGroup00-my--lv
            vg_name, lv_name = re.search("(.*[^-])-([^-].*)", name).groups()
            vg_name = vg_name.replace("--", "-")
            lv_name = lv_name.replace("--", "-")

            if not lv_name in lvs[vg_name]:
                # This isn't something we saw as a named LV, so its
                # a partition.  Assign its parent and don't store it
                # as an LV.
                result = re.search("(.*)p\d+", lv_name)
                if not result:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                    continue
                parent_lv_name = result.groups()[0]
                if not parent_lv_name in lvs[vg_name]:
                    agent_log.error("Cannot parse LVM device name %s" % name)
                else:
                    # FIXME: compose path in a way that copes with hyphens
                    parent_block_device = node_block_devices["/dev/mapper/%s-%s" % (vg_name, parent_lv_name)]
                    block_device_nodes[block_device]['parent'] = parent_block_device

                continue
            lvs[vg_name][lv_name]['block_device'] = block_device

            devices = [block_device_nodes[i]['major_minor'] for i in devices]
            vgs[vg_name]['pvs_major_minor'] = list(set(vgs[vg_name]['pvs_major_minor']) | set(devices))
        elif dm_type == 'multipath':
            # This is a multipath device, there will be a list of devices like this:
            # "round-robin 0 1 1 8:80 1000"
            dev_indices = range(12, len(tokens), 6)
            devices = [tokens[i] for i in dev_indices]
            devices = [block_device_nodes[i] for i in devices]
            if name in mpaths:
                raise RuntimeError("Duplicated mpath device %s" % name)

            mpaths[name] = {
                    "name": name,
                    "block_device": block_device,
                    "nodes": devices
                    }
        else:
            continue

    # Anything in fstab or that is mounted
    from hydra_agent.legacy_audit import Fstab, Mounts
    fstab = Fstab()
    mounts = Mounts()
    from itertools import chain
    bdev_to_local_fs = {}
    for dev, mntpnt, fstype in chain(fstab.all(), mounts.all()):
        mm = _dev_major_minor(dev)
        if mm and fstype != 'lustre' and mm in block_device_nodes:
            bdev_to_local_fs[mm] = (mntpnt, fstype)

    return {"vgs": vgs, "lvs": lvs, "mpath": mpaths, "devs": block_device_nodes, "local_fs": bdev_to_local_fs}
<|endoftext|>


<commit_before>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""
from toolbasics import *
from albow.dialogs import wrapped_label

def alertFilterException(func):
    def _func(*args, **kw):
        try:
            func(*args, **kw)
        except Exception, e:
            alert(u"Exception during filter operation. See console for details.\n\n{0}".format(e))
            print traceback.format_exc()

    return _func

class FilterModuleOptions(Widget):
    is_gl_container = True

    def __init__(self, tool, module, *args, **kw):
        Widget.__init__(self, *args, **kw)
        rows = []
        self.optionDict = {}
        self.tool = tool
        cols = []
        height = 0
        max_height = 550

        print "Creating options for ", module
        if hasattr(module, "inputs"):

            for optionName, optionType in module.inputs:
                if isinstance(optionType, tuple):
                    if isinstance(optionType[0], (int, long, float)):
                        if len(optionType) > 2:
                            val, min, max = optionType
                        elif len(optionType) == 2:
                            min, max = optionType
                            val = min

                        rows.append(self.addNumberField(optionName, val, min, max))
                    if isinstance(optionType[0], (str, unicode)):
                        choiceButton = ChoiceButton(map(str, optionType))
                        self.optionDict[optionName] = AttrRef(choiceButton, 'selectedChoice')

                        rows.append(Row((Label(optionName), choiceButton)))

                elif isinstance(optionType, bool):
                    cbox = CheckBox(value=optionType)
                    self.optionDict[optionName] = AttrRef(cbox, 'value')

                    row = Row((Label(optionName), cbox))
                    rows.append(row)

                elif isinstance(optionType, (int, float)):
                    rows.append(self.addNumberField(optionName, optionType))

                elif optionType == "blocktype" or isinstance(optionType, materials.Block):
                    blockButton = BlockButton(self.tool.editor.level.materials)
                    if isinstance(optionType, materials.Block):
                        blockButton.blockInfo = optionType

                    row = Column((Label(optionName), blockButton))
                    self.optionDict[optionName] = AttrRef(blockButton, 'blockInfo')

                    rows.append(row)
                elif optionType == "label":
                    rows.append(wrapped_label(optionName, 50))

                else:
                    raise ValueError(("Unknown option type", optionType))

            height = sum(r.height for r in rows)

            if height > max_height:
                h = 0
                for i, r in enumerate(rows):
                    h += r.height
                    if h > height / 2:
                        break

                cols.append(Column(rows[:i]))
                rows = rows[i:]
            #cols.append(Column(rows))

            if len(rows):
                cols.append(Column(rows))

            if len(cols):
                self.add(Row(cols))
            self.shrink_wrap()
        else:
            self.size = (0, 0)

    def addNumberField(self, optionName, val, min=None, max=None):
        if isinstance(val, float):
            ftype = FloatField
        else:
            ftype = IntField

        if min == max:
            min = None
            max = None

        field = ftype(value=val, width=100, min=min, max=max)
        self.optionDict[optionName] = AttrRef(field, 'value')

        row = Row([Label(optionName), field])
        return row

    @property
    def options(self):
        return dict((k, v.get()) for k, v in self.optionDict.iteritems())

    @options.setter
    def options(self, val):
        for k in val:
            if k in self.optionDict:
                self.optionDict[k].set(val[k])

class FilterToolPanel(Panel):
    def __init__(self, tool):
        Panel.__init__(self)

        self.savedOptions = {}

        self.tool = tool
        self.selectedFilterName = None
        if len(self.tool.filterModules):
            self.reload()

    def reload(self):
        for i in list(self.subwidgets):
            self.remove(i)

        tool = self.tool

        if len(tool.filterModules) is 0:
            self.add(Label("No filter modules found!"))
            self.shrink_wrap()
            return

        if self.selectedFilterName is None or self.selectedFilterName not in tool.filterNames:
            self.selectedFilterName = tool.filterNames[0]

        self.filterOptionsPanel = None
        while self.filterOptionsPanel is None:
            module = self.tool.filterModules[self.selectedFilterName]
            try:
                self.filterOptionsPanel = FilterModuleOptions(self.tool, module)
            except Exception, e:
                alert("Error creating filter inputs for {0}: {1}".format(module, e))
                traceback.print_exc()
                self.tool.filterModules.pop(self.selectedFilterName)
                self.selectedFilterName = tool.filterNames[0]

            if len(tool.filterNames) == 0:
                raise ValueError("No filters loaded!")

        self.filterSelect = ChoiceButton(tool.filterNames, choose=self.filterChanged)
        self.filterSelect.selectedChoice = self.selectedFilterName

        self.confirmButton = Button("Filter", action=self.tool.confirm)

        filterLabel = Label("Filter:", fg_color=(177, 177, 255, 255))
        filterLabel.mouse_down = lambda x: mcplatform.platform_open(mcplatform.filtersDir)
        filterLabel.tooltipText = "Click to open filters folder"
        filterSelectRow = Row((filterLabel, self.filterSelect))

        self.add(Column((filterSelectRow, self.filterOptionsPanel, self.confirmButton)))

        self.shrink_wrap()
        if self.parent:
            self.centery = self.parent.centery

        if self.selectedFilterName in self.savedOptions:
            self.filterOptionsPanel.options = self.savedOptions[self.selectedFilterName]

    def filterChanged(self):
        self.saveOptions()
        self.selectedFilterName = self.filterSelect.selectedChoice
        self.reload()

    filterOptionsPanel = None

    def saveOptions(self):
        if self.filterOptionsPanel:
            self.savedOptions[self.selectedFilterName] = self.filterOptionsPanel.options

class FilterOperation(Operation):
    def __init__(self, level, box, filter, options):
        self.box = box
        self.level = level
        self.filter = filter
        self.options = options

    def perform(self, recordUndo=True):
        if recordUndo:
            self.recordUndo()

        self.filter.perform(self.level, BoundingBox(self.box), self.options)

        pass

    def recordUndo(self):
        self.undoSchematic = self.extractUndoSchematicFrom(self.level, self.box)

    def undo(self):
        if self.undoSchematic:
            self.level.removeEntitiesInBox(self.box)
            self.level.removeTileEntitiesInBox(self.box)
            self.level.copyBlocksFrom(self.undoSchematic, BoundingBox((0, 0, 0), self.box.size), self.box.origin)

    def dirtyBox(self):
        return self.box

class FilterTool(EditorTool):
    tooltipText = "Filter"
    toolIconName = "filter"

    def __init__(self, editor):
        EditorTool.__init__(self, editor)

        self.filterModules = {}

        self.panel = FilterToolPanel(self)

    @property
    def statusText(self):
        return "Choose a filter, then click Filter or press ENTER to apply it."

    def toolEnabled(self):
        return not (self.selectionBox() is None)

    def toolSelected(self):
        self.showPanel()

    @alertException
    def showPanel(self):
        if self.panel.parent:
            self.editor.remove(self.panel)

        self.reloadFilters()
        #self.panel = FilterToolPanel(self)
        self.panel.reload()

        self.panel.left = self.editor.left
        self.panel.centery = self.editor.centery

        self.editor.add(self.panel)

    def hidePanel(self):
        self.panel.saveOptions()
        if self.panel.parent:
            self.panel.parent.remove(self.panel)

    def reloadFilters(self):
        filterDir = mcplatform.filtersDir
        filterFiles = os.listdir(filterDir)
        filterPyfiles = filter(lambda x: x.endswith(".py"), filterFiles)

        filterModules = (__import__(x[:-3]) for x in filterPyfiles)
        filterModules = filter(lambda module: hasattr(module, "perform"), filterModules)

        self.filterModules = dict((self.moduleDisplayName(x), x) for x in filterModules)
        [reload(m) for m in self.filterModules.itervalues()]
        filterModules = (__import__(x[:-3]) for x in filterPyfiles)

    @property
    def filterNames(self):
        return [self.moduleDisplayName(module) for module in self.filterModules.itervalues()]

    def moduleDisplayName(self, module):
        return module.displayName if hasattr(module, 'displayName') else module.__name__.capitalize()

    @alertFilterException
    def confirm(self):

        with setWindowCaption("APPLYING FILTER - "):
            filterModule = self.filterModules[self.panel.filterSelect.selectedChoice]

            op = FilterOperation(self.editor.level, self.selectionBox(), filterModule, self.panel.filterOptionsPanel.options)

            self.editor.level.showProgress = showProgress
            self.performWithRetry(op)

            self.editor.addOperation(op)
            self.editor.addUnsavedEdit()

            self.editor.invalidateBox(self.selectionBox())
<commit_msg>Update filter.py to Allow String Input for Filters<commit_after>"""Copyright (c) 2010-2012 David Rio Vierra

Permission to use, copy, modify, and/or distribute this software for any
purpose with or without fee is hereby granted, provided that the above
copyright notice and this permission notice appear in all copies.

THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE."""
from toolbasics import *
from albow.dialogs import wrapped_label

def alertFilterException(func):
    def _func(*args, **kw):
        try:
            func(*args, **kw)
        except Exception, e:
            alert(u"Exception during filter operation. See console for details.\n\n{0}".format(e))
            print traceback.format_exc()

    return _func

class FilterModuleOptions(Widget):
    is_gl_container = True

    def __init__(self, tool, module, *args, **kw):
        Widget.__init__(self, *args, **kw)
        rows = []
        self.optionDict = {}
        self.tool = tool
        cols = []
        height = 0
        max_height = 550

        print "Creating options for ", module
        if hasattr(module, "inputs"):

            for optionName, optionType in module.inputs:
                if isinstance(optionType, tuple):
                    if isinstance(optionType[0], (int, long, float)):
                        if len(optionType) > 2:
                            val, min, max = optionType
                        elif len(optionType) == 2:
                            min, max = optionType
                            val = min

                        rows.append(self.addNumberField(optionName, val, min, max))
                    if isinstance(optionType[0], (str, unicode)):
                        choiceButton = ChoiceButton(map(str, optionType))
                        self.optionDict[optionName] = AttrRef(choiceButton, 'selectedChoice')

                        rows.append(Row((Label(optionName), choiceButton)))

                elif isinstance(optionType, bool):
                    cbox = CheckBox(value=optionType)
                    self.optionDict[optionName] = AttrRef(cbox, 'value')

                    row = Row((Label(optionName), cbox))
                    rows.append(row)

                elif isinstance(optionType, (int, float)):
                    rows.append(self.addNumberField(optionName, optionType))

                elif optionType == "blocktype" or isinstance(optionType, materials.Block):
                    blockButton = BlockButton(self.tool.editor.level.materials)
                    if isinstance(optionType, materials.Block):
                        blockButton.blockInfo = optionType

                    row = Column((Label(optionName), blockButton))
                    self.optionDict[optionName] = AttrRef(blockButton, 'blockInfo')

                    rows.append(row)
                elif optionType == "label":
                    rows.append(wrapped_label(optionName, 50))

                elif optionType == "string":
                    field = TextField(value="string")
                    self.optionDict[optionName] = AttrRef(field, 'value')

                    row = Row((Label(optionName), field))
                    rows.append(row)

                else:
                    raise ValueError(("Unknown option type", optionType))

            height = sum(r.height for r in rows)

            if height > max_height:
                h = 0
                for i, r in enumerate(rows):
                    h += r.height
                    if h > height / 2:
                        break

                cols.append(Column(rows[:i]))
                rows = rows[i:]
            #cols.append(Column(rows))

            if len(rows):
                cols.append(Column(rows))

            if len(cols):
                self.add(Row(cols))
            self.shrink_wrap()
        else:
            self.size = (0, 0)

    def addNumberField(self, optionName, val, min=None, max=None):
        if isinstance(val, float):
            ftype = FloatField
        else:
            ftype = IntField

        if min == max:
            min = None
            max = None

        field = ftype(value=val, width=100, min=min, max=max)
        self.optionDict[optionName] = AttrRef(field, 'value')

        row = Row([Label(optionName), field])
        return row

    @property
    def options(self):
        return dict((k, v.get()) for k, v in self.optionDict.iteritems())

    @options.setter
    def options(self, val):
        for k in val:
            if k in self.optionDict:
                self.optionDict[k].set(val[k])

class FilterToolPanel(Panel):
    def __init__(self, tool):
        Panel.__init__(self)

        self.savedOptions = {}

        self.tool = tool
        self.selectedFilterName = None
        if len(self.tool.filterModules):
            self.reload()

    def reload(self):
        for i in list(self.subwidgets):
            self.remove(i)

        tool = self.tool

        if len(tool.filterModules) is 0:
            self.add(Label("No filter modules found!"))
            self.shrink_wrap()
            return

        if self.selectedFilterName is None or self.selectedFilterName not in tool.filterNames:
            self.selectedFilterName = tool.filterNames[0]

        self.filterOptionsPanel = None
        while self.filterOptionsPanel is None:
            module = self.tool.filterModules[self.selectedFilterName]
            try:
                self.filterOptionsPanel = FilterModuleOptions(self.tool, module)
            except Exception, e:
                alert("Error creating filter inputs for {0}: {1}".format(module, e))
                traceback.print_exc()
                self.tool.filterModules.pop(self.selectedFilterName)
                self.selectedFilterName = tool.filterNames[0]

            if len(tool.filterNames) == 0:
                raise ValueError("No filters loaded!")

        self.filterSelect = ChoiceButton(tool.filterNames, choose=self.filterChanged)
        self.filterSelect.selectedChoice = self.selectedFilterName

        self.confirmButton = Button("Filter", action=self.tool.confirm)

        filterLabel = Label("Filter:", fg_color=(177, 177, 255, 255))
        filterLabel.mouse_down = lambda x: mcplatform.platform_open(mcplatform.filtersDir)
        filterLabel.tooltipText = "Click to open filters folder"
        filterSelectRow = Row((filterLabel, self.filterSelect))

        self.add(Column((filterSelectRow, self.filterOptionsPanel, self.confirmButton)))

        self.shrink_wrap()
        if self.parent:
            self.centery = self.parent.centery

        if self.selectedFilterName in self.savedOptions:
            self.filterOptionsPanel.options = self.savedOptions[self.selectedFilterName]

    def filterChanged(self):
        self.saveOptions()
        self.selectedFilterName = self.filterSelect.selectedChoice
        self.reload()

    filterOptionsPanel = None

    def saveOptions(self):
        if self.filterOptionsPanel:
            self.savedOptions[self.selectedFilterName] = self.filterOptionsPanel.options

class FilterOperation(Operation):
    def __init__(self, level, box, filter, options):
        self.box = box
        self.level = level
        self.filter = filter
        self.options = options

    def perform(self, recordUndo=True):
        if recordUndo:
            self.recordUndo()

        self.filter.perform(self.level, BoundingBox(self.box), self.options)

        pass

    def recordUndo(self):
        self.undoSchematic = self.extractUndoSchematicFrom(self.level, self.box)

    def undo(self):
        if self.undoSchematic:
            self.level.removeEntitiesInBox(self.box)
            self.level.removeTileEntitiesInBox(self.box)
            self.level.copyBlocksFrom(self.undoSchematic, BoundingBox((0, 0, 0), self.box.size), self.box.origin)

    def dirtyBox(self):
        return self.box

class FilterTool(EditorTool):
    tooltipText = "Filter"
    toolIconName = "filter"

    def __init__(self, editor):
        EditorTool.__init__(self, editor)

        self.filterModules = {}

        self.panel = FilterToolPanel(self)

    @property
    def statusText(self):
        return "Choose a filter, then click Filter or press ENTER to apply it."

    def toolEnabled(self):
        return not (self.selectionBox() is None)

    def toolSelected(self):
        self.showPanel()

    @alertException
    def showPanel(self):
        if self.panel.parent:
            self.editor.remove(self.panel)

        self.reloadFilters()
        #self.panel = FilterToolPanel(self)
        self.panel.reload()

        self.panel.left = self.editor.left
        self.panel.centery = self.editor.centery

        self.editor.add(self.panel)

    def hidePanel(self):
        self.panel.saveOptions()
        if self.panel.parent:
            self.panel.parent.remove(self.panel)

    def reloadFilters(self):
        filterDir = mcplatform.filtersDir
        filterFiles = os.listdir(filterDir)
        filterPyfiles = filter(lambda x: x.endswith(".py"), filterFiles)

        filterModules = (__import__(x[:-3]) for x in filterPyfiles)
        filterModules = filter(lambda module: hasattr(module, "perform"), filterModules)

        self.filterModules = dict((self.moduleDisplayName(x), x) for x in filterModules)
        [reload(m) for m in self.filterModules.itervalues()]
        filterModules = (__import__(x[:-3]) for x in filterPyfiles)

    @property
    def filterNames(self):
        return [self.moduleDisplayName(module) for module in self.filterModules.itervalues()]

    def moduleDisplayName(self, module):
        return module.displayName if hasattr(module, 'displayName') else module.__name__.capitalize()

    @alertFilterException
    def confirm(self):

        with setWindowCaption("APPLYING FILTER - "):
            filterModule = self.filterModules[self.panel.filterSelect.selectedChoice]

            op = FilterOperation(self.editor.level, self.selectionBox(), filterModule, self.panel.filterOptionsPanel.options)

            self.editor.level.showProgress = showProgress
            self.performWithRetry(op)

            self.editor.addOperation(op)
            self.editor.addUnsavedEdit()

            self.editor.invalidateBox(self.selectionBox())
<|endoftext|>


<commit_before>from django.apps import apps
from django_dynamic_fixture import G
from .. import base_tests

MediaCategory = apps.get_model('icekit.MediaCategory')

class MediaCategoryAPITests(base_tests._BaseAPITestCase):
    API_NAME = 'media-category-api'

    def setUp(self):
        super(MediaCategoryAPITests, self).setUp()

        self.media_category = G(
            MediaCategory,
            name='Test media category',
        )

    def test_list_media_categories_with_get(self):
        response = self.client.get(self.listing_url())
        self.assertEqual(200, response.status_code)
        self.assertEqual(
            {
                'count': 1,
                'next': None,
                'previous': None,
                'results': [{
                    'id': self.media_category.pk,
                    'name': 'Test media category',
                }]
            },
            response.data)

    def test_get_media_category_detail_with_get(self):
        response = self.client.get(self.detail_url(self.media_category.pk))
        self.assertEqual(200, response.status_code)
        self.assertEqual(
            {
                'id': self.media_category.pk,
                'name': 'Test media category',
            },
            response.data)

    def test_add_media_category_with_post(self):
        response = self.client.post(
            self.listing_url(),
            {
                'name': 'New media category',
            },
        )
        self.assertEqual(201, response.status_code)
        new_media_category = MediaCategory.objects.get(pk=response.data['id'])
        self.assertEqual('New media category', new_media_category.name)

    def test_replace_media_category_with_put(self):
        response = self.client.get(self.detail_url(self.media_category.pk))
        self.assertEqual(200, response.status_code)

        media_category_data = response.data
        media_category_data['name'] = 'Replaced media category'

        response = self.client.put(
            self.detail_url(self.media_category.pk),
            media_category_data,
        )
        self.assertEqual(200, response.status_code)
        updated_media_category = MediaCategory.objects.get(pk=self.media_category.pk)
        self.assertEqual('Replaced media category', updated_media_category.name)

    def test_update_media_category_with_patch(self):
        response = self.client.patch(
            self.detail_url(self.media_category.pk),
            {'title': 'Updated media category'},
        )
        self.assertEqual(200, response.status_code)
        updated_media_category = MediaCategory.objects.get(pk=self.media_category.pk)
        self.assertEqual('Updated media category', updated_media_category.name)

    def test_delete_media_category_with_delete(self):
        response = self.client.delete(self.detail_url(self.media_category.pk))
        self.assertEqual(204, response.status_code)
        self.assertEqual(0, MediaCategory.objects.count())

    def test_api_user_permissions_are_correct(self):
        self.assert_api_user_permissions_are_correct(
            self.media_category.pk,
            MediaCategory,
            'mediacategory',
        )
<commit_msg>MediaCategory API test fix<commit_after>from django.apps import apps
from django_dynamic_fixture import G
from .. import base_tests

MediaCategory = apps.get_model('icekit.MediaCategory')

class MediaCategoryAPITests(base_tests._BaseAPITestCase):
    API_NAME = 'media-category-api'

    def setUp(self):
        super(MediaCategoryAPITests, self).setUp()

        self.media_category = G(
            MediaCategory,
            name='Test media category',
        )

    def test_list_media_categories_with_get(self):
        response = self.client.get(self.listing_url())
        self.assertEqual(200, response.status_code)
        self.assertEqual(
            {
                'count': 1,
                'next': None,
                'previous': None,
                'results': [{
                    'id': self.media_category.pk,
                    'name': 'Test media category',
                }]
            },
            response.data)

    def test_get_media_category_detail_with_get(self):
        response = self.client.get(self.detail_url(self.media_category.pk))
        self.assertEqual(200, response.status_code)
        self.assertEqual(
            {
                'id': self.media_category.pk,
                'name': 'Test media category',
            },
            response.data)

    def test_add_media_category_with_post(self):
        response = self.client.post(
            self.listing_url(),
            {
                'name': 'New media category',
            },
        )
        self.assertEqual(201, response.status_code)
        new_media_category = MediaCategory.objects.get(pk=response.data['id'])
        self.assertEqual('New media category', new_media_category.name)

    def test_replace_media_category_with_put(self):
        response = self.client.get(self.detail_url(self.media_category.pk))
        self.assertEqual(200, response.status_code)

        media_category_data = response.data
        media_category_data['name'] = 'Replaced media category'

        response = self.client.put(
            self.detail_url(self.media_category.pk),
            media_category_data,
        )
        self.assertEqual(200, response.status_code)
        updated_media_category = MediaCategory.objects.get(pk=self.media_category.pk)
        self.assertEqual('Replaced media category', updated_media_category.name)

    def test_update_media_category_with_patch(self):
        response = self.client.patch(
            self.detail_url(self.media_category.pk),
            {'name': 'Updated media category'},
        )
        self.assertEqual(200, response.status_code)
        updated_media_category = MediaCategory.objects.get(pk=self.media_category.pk)
        self.assertEqual('Updated media category', updated_media_category.name)

    def test_delete_media_category_with_delete(self):
        response = self.client.delete(self.detail_url(self.media_category.pk))
        self.assertEqual(204, response.status_code)
        self.assertEqual(0, MediaCategory.objects.count())

    def test_api_user_permissions_are_correct(self):
        self.assert_api_user_permissions_are_correct(
            self.media_category.pk,
            MediaCategory,
            'mediacategory',
        )
<|endoftext|>