<commit_before>import sys
import json
import numpy as np
from sklearn.cross_validation import KFold
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.utils import shuffle

from ambra.classifiers import IntervalLogisticRegression
from ambra.interval_scoring import semeval_interval_scorer
from sklearn.feature_extraction.text import TfidfVectorizer

class MyPipeline(Pipeline):
    def predict(self, X, Y_possible):
            """Applies transforms to the data, and the predict method of the
            final estimator. Valid only if the final estimator implements
            predict."""
            Xt = X
            for name, transform in self.steps[:-1]:
                Xt = transform.transform(Xt)
            return self.steps[-1][-1].predict(Xt, Y_possible)

with open(sys.argv[1]) as f:
    entries = json.load(f)

X_len = np.array([len(doc['text'].split()) for doc in entries])[:, np.newaxis]
X_txt = np.array([doc['text'] for doc in entries])
Y = np.array([doc['true_interval'] for doc in entries])
Y_possible = np.array([doc['possible_intervals'] for doc in entries])

X_len, X_txt, Y, Y_possible = shuffle(X_len, X_txt, Y, Y_possible,
                                      random_state=0)

# make it run fast
X_len = X_len[:100]
X_txt = X_txt[:100]
Y = Y[:100]
Y_possible = Y_possible[:100]

grid = GridSearchCV(IntervalLogisticRegression(C=1.0),
                    dict(n_neighbors=(5, 10, 50)),
                    verbose=True, cv=KFold(len(X_len), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible))

grid.fit(X_len, Y)
print grid.best_score_, grid.best_params_

vect = TfidfVectorizer(max_df=0.95, max_features=50, use_idf=False, norm="l1")
pipe = MyPipeline([('vect', vect),
                   ('clf', IntervalLogisticRegression(n_neighbors=10))])

grid = GridSearchCV(pipe, dict(clf__C=[0.01, 1.0, 10.0]),
                    verbose=True, cv=KFold(len(X_len), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible))

grid.fit(X_txt, Y)
print grid.best_score_, grid.best_params_
<commit_msg>Proof of concept feature concatenation<commit_after>import sys
import json
import numpy as np
from sklearn.cross_validation import KFold
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.grid_search import GridSearchCV
from sklearn.utils import shuffle
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.preprocessing import StandardScaler

from ambra.classifiers import IntervalLogisticRegression
from ambra.interval_scoring import semeval_interval_scorer

class MyPipeline(Pipeline):
    def predict(self, X, Y_possible):
            """Applies transforms to the data, and the predict method of the
            final estimator. Valid only if the final estimator implements
            predict."""
            Xt = X
            for name, transform in self.steps[:-1]:
                Xt = transform.transform(Xt)
            return self.steps[-1][-1].predict(Xt, Y_possible)

class Proj(BaseEstimator, TransformerMixin):
    """Projection metaestimator for list-of-dict kind of data"""
    def __init__(self, transf, key):
        self.transf = transf
        self.key = key

    def fit(self, X, y=None):
        return self.transf.fit([x[self.key] for x in X], y)

    def transform(self, X):
        return self.transf.transform([x[self.key] for x in X])

    def fit_transform(self, X, y=None):
        return self.transf.fit_transform([x[self.key] for x in X], y)

    def get_feature_names(self):
        return self.transf.get_feature_names()

class LengthFeatures(BaseEstimator, TransformerMixin):
    def fit(self, X, y=None):
        return self

    def _doc_features(self, doc):
        n_sents = len(doc)
        all_toks = [tok for sent in doc for tok in sent]
        n_tokens = len(all_toks)
        n_types = len(set(all_toks))
        type_token_ratio = n_tokens / float(n_types)
        return np.array([n_sents, n_tokens, n_types, type_token_ratio],
                        dtype=np.float)

    def transform(self, X, y=None):
        #x in X is a list of sents
        return np.row_stack([self._doc_features(doc) for doc in X])

    def get_feature_names(self):
        return ['n_sents', 'n_tokens', 'n_types', 'type_token_ratio']

class NgramLolAnalyzer(object):
    def _word_ngrams(self, tokens, stop_words=None):
        """Turn tokens into a sequence of n-grams after stop words filtering"""
        # handle stop words
        if stop_words is not None:
            tokens = [w for w in tokens if w not in stop_words]
        if self.lower:
            tokens = [w.lower() for w in tokens]
        # handle token n-grams
        min_n, max_n = self.ngram_range
        if max_n != 1:
            original_tokens = tokens
            tokens = []
            n_original_tokens = len(original_tokens)
            for n in xrange(min_n,
                            min(max_n + 1, n_original_tokens + 1)):
                for i in xrange(n_original_tokens - n + 1):
                    tokens.append(" ".join(original_tokens[i: i + n]))
        return tokens

    def __init__(self, ngram_range, lower=False):
        self.ngram_range=ngram_range
        self.lower = lower

    def __call__(self, doc):
        return [feature for sentence in doc
                for feature in self._word_ngrams(sentence)]

with open(sys.argv[1]) as f:
    entries = json.load(f)

# some buggy docs are empty
entries = [entry for entry in entries if len(entry['lemmas'])]

X = np.array(entries)
Y = np.array([doc['interval'] for doc in entries])
Y_possible = np.array([doc['all_fine_intervals'] for doc in entries])

X, Y, Y_possible = shuffle(X, Y, Y_possible, random_state=0)

# make it run fast
X = X[:100]
Y = Y[:100]
Y_possible = Y_possible[:100]

pipe = MyPipeline([('vect', Proj(LengthFeatures(), key='lemmas')),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10))])

grid = GridSearchCV(pipe,
                    dict(clf__C=np.logspace(-3, 3, 3)),
                    verbose=True, cv=KFold(len(X), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible),
                    n_jobs=1)

grid.fit(X, Y)
print grid.best_score_, grid.best_params_
print grid.best_estimator_.steps[-1][-1].coef_.ravel()

pos_vect = TfidfVectorizer(use_idf=False, norm='l1', min_df=5, max_df=0.9,
                           analyzer=NgramLolAnalyzer(ngram_range=(2, 3),
                                                     lower=False))

union = FeatureUnion([('lenghts', Proj(LengthFeatures(), key='lemmas')),
                      ('pos', Proj(pos_vect, key='pos'))])

pipe = MyPipeline([('union', union),
                   ('scale', StandardScaler(with_mean=False, with_std=True)),
                   ('clf', IntervalLogisticRegression(n_neighbors=10))])

grid = GridSearchCV(pipe,
                    dict(clf__C=np.logspace(-3, 3, 3)),
                    verbose=True, cv=KFold(len(X), n_folds=5),
                    scoring=semeval_interval_scorer,
                    scorer_params=dict(Y_possible=Y_possible),
                    n_jobs=1)

grid.fit(X, Y)
print grid.best_score_, grid.best_params_

feature_names = grid.best_estimator_.steps[0][1].get_feature_names()
coef = grid.best_estimator_.steps[-1][-1].coef_.ravel()

for idx in np.argsort(-coef)[:10]:
    print("{:.2f}\t{}".format(coef[idx], feature_names[idx]))

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Flow Framework documentation build configuration file, created by
# sphinx-quickstart on Mon Jun 08 11:09:23 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Flow Framework'
copyright = u'2006 and onwards by the authors'
author = u'Neos Team and Contributors'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '6.0'
# The full version, including alpha/beta/rc tags.
release = '6.0.x-dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'FlowFrameworkdoc'

# Configure links to view/edit source to point to development collection
github_user' = 'neos'
github_repo' = 'flow-development-collection'
github_version' = '5.3'
conf_py_path = '/Neos.Flow/Documentation/'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
 'papersize': 'a4paper',

# The font size ('10pt', '11pt' or '12pt').
 'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'FlowFramework.tex', u'Flow Framework Documentation',
   u'The Neos Team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_use_parts = True

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'flowframework', u'Flow Framework Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'FlowFramework', u'Flow Framework Documentation',
   author, 'Flow Framework', 'A PHP Framework, built to power Neos CMS',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'neos': ('https://neos.readthedocs.io/en/stable', None),
}

# load PhpLexer
from sphinx.highlighting import lexers
from pygments.lexers.web import PhpLexer

# enable highlighting for PHP code not between <?php ... ?> by default
lexers['php'] = PhpLexer(startinline=True)
lexers['php-annotations'] = PhpLexer(startinline=True)

# Use PHP syntax highlighting in code examples by default
highlight_language='php'
<commit_msg>TASK: Fix github_version<commit_after># -*- coding: utf-8 -*-
#
# Flow Framework documentation build configuration file, created by
# sphinx-quickstart on Mon Jun 08 11:09:23 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import sys
import os
import shlex

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
#needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.intersphinx',
    'sphinx.ext.todo',
    'sphinx.ext.ifconfig'
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'

# The encoding of source files.
#source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Flow Framework'
copyright = u'2006 and onwards by the authors'
author = u'Neos Team and Contributors'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '6.0'
# The full version, including alpha/beta/rc tags.
release = '6.0.x-dev'

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
#today = ''
# Else, today_fmt is used as the format for a strftime call.
#today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = ['_build']

# The reST default role (used for this markup: `text`) to use for all
# documents.
#default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
#add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
#add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
#show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# A list of ignored prefixes for module index sorting.
#modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
#keep_warnings = False

# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False

# -- Options for HTML output ----------------------------------------------

# on_rtd is whether we are on readthedocs.org, this line of code grabbed from docs.readthedocs.org
on_rtd = os.environ.get('READTHEDOCS', None) == 'True'

if not on_rtd:  # only import and set the theme if we're building docs locally
    import sphinx_rtd_theme
    html_theme = 'sphinx_rtd_theme'
    html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]

# otherwise, readthedocs.org uses their theme by default, so no need to specify it

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
# html_theme = 'alabaster'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
#html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
#html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
#html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
#html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
#html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
#html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
#html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
#html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
#html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
#html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
#html_additional_pages = {}

# If false, no module index is generated.
#html_domain_indices = True

# If false, no index is generated.
#html_use_index = True

# If true, the index is split into individual pages for each letter.
#html_split_index = False

# If true, links to the reST sources are added to the pages.
#html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
#html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
#html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
#html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
#html_file_suffix = None

# Language to be used for generating the HTML full-text search index.
# Sphinx supports the following languages:
#   'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja'
#   'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr'
#html_search_language = 'en'

# A dictionary with options for the search language support, empty by default.
# Now only 'ja' uses this config value
#html_search_options = {'type': 'default'}

# The name of a javascript file (relative to the configuration directory) that
# implements a search results scorer. If empty, the default will be used.
#html_search_scorer = 'scorer.js'

# Output file base name for HTML help builder.
htmlhelp_basename = 'FlowFrameworkdoc'

# Configure links to view/edit source to point to development collection
github_user' = 'neos'
github_repo' = 'flow-development-collection'
github_version' = '6.0'
conf_py_path = '/Neos.Flow/Documentation/'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
 'papersize': 'a4paper',

# The font size ('10pt', '11pt' or '12pt').
 'pointsize': '10pt',

# Additional stuff for the LaTeX preamble.
#'preamble': '',

# Latex figure (float) alignment
#'figure_align': 'htbp',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
  (master_doc, 'FlowFramework.tex', u'Flow Framework Documentation',
   u'The Neos Team', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
#latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
latex_use_parts = True

# If true, show page references after internal links.
#latex_show_pagerefs = False

# If true, show URL addresses after external links.
#latex_show_urls = False

# Documents to append as an appendix to all manuals.
#latex_appendices = []

# If false, no module index is generated.
#latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    (master_doc, 'flowframework', u'Flow Framework Documentation',
     [author], 1)
]

# If true, show URL addresses after external links.
#man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
  (master_doc, 'FlowFramework', u'Flow Framework Documentation',
   author, 'Flow Framework', 'A PHP Framework, built to power Neos CMS',
   'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
#texinfo_appendices = []

# If false, no module index is generated.
#texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
#texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
#texinfo_no_detailmenu = False

# Example configuration for intersphinx: refer to the Python standard library.
intersphinx_mapping = {
    'neos': ('https://neos.readthedocs.io/en/stable', None),
}

# load PhpLexer
from sphinx.highlighting import lexers
from pygments.lexers.web import PhpLexer

# enable highlighting for PHP code not between <?php ... ?> by default
lexers['php'] = PhpLexer(startinline=True)
lexers['php-annotations'] = PhpLexer(startinline=True)

# Use PHP syntax highlighting in code examples by default
highlight_language='php'
<|endoftext|>


<commit_before>from nodeconductor.core import NodeConductorExtension

class OracleExtension(NodeConductorExtension):

    class Settings:
        ORACLE_TICKET_TEMPLATES = {
            'provision': {
                'summary': "Request for a new Oracle DB PaaS instance",
                'details': """
                    Oracle DB purchase details

                    Customer name: {customer.name}
                    Project name: {project.project_group.name}
                    Environment name: {project.name}
                    Customer UUID: {customer.uuid.hex}
                    Project UUID: {project.project_group.uuid.hex}
                    Environment UUID: {project.uuid.hex}
                    OpenStack tenant id: {deployment.tenant.uuid.hex}

                    Hardware Configuration:
                    Name: {deployment.name}
                    Flavor: {deployment.flavor_info}
                    SSH key: {ssh_key.name}
                    SSH key UUID: {ssh_key.uuid.hex}

                    Oracle DB Configuration:
                    Name: {deployment.db_name}
                    Size: {deployment.db_size} GB / {deployment.db_arch_size} GB
                    Version: {deployment.db_version_type}
                    Database type: {deployment.db_template}
                    Character set: {deployment.db_charset}
                    Additional data: {deployment.user_data}
                """,
            },
            'undeploy': {
                'summary': "Request for removing Oracle DB PaaS instance",
                'details': """
                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}
                """,
            },
            'resize': {
                'summary': "Request for resizing Oracle DB PaaS instance",
                'details': """
                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}

                    Hardware Configuration:
                    Flavor: {deployment.flavor_info}
                """,
            },
            'support': {
                'summary': "Custom support request",
                'details': """
                    {message}

                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}
                """,
            },
        }

    @staticmethod
    def django_app():
        return 'nodeconductor_paas_oracle'

    @staticmethod
    def rest_urls():
        from .urls import register_in
        return register_in
<commit_msg>Extend template for custom operations with more details<commit_after>from nodeconductor.core import NodeConductorExtension

class OracleExtension(NodeConductorExtension):

    class Settings:
        ORACLE_TICKET_TEMPLATES = {
            'provision': {
                'summary': "Request for a new Oracle DB PaaS instance",
                'details': """
                    Oracle DB purchase details

                    Customer name: {customer.name}
                    Project name: {project.project_group.name}
                    Environment name: {project.name}
                    Customer UUID: {customer.uuid.hex}
                    Project UUID: {project.project_group.uuid.hex}
                    Environment UUID: {project.uuid.hex}
                    OpenStack tenant id: {deployment.tenant.uuid.hex}

                    Hardware Configuration:
                    Name: {deployment.name}
                    Flavor: {deployment.flavor_info}
                    SSH key: {ssh_key.name}
                    SSH key UUID: {ssh_key.uuid.hex}

                    Oracle DB Configuration:
                    Name: {deployment.db_name}
                    Size: {deployment.db_size} GB / {deployment.db_arch_size} GB
                    Version: {deployment.db_version_type}
                    Database type: {deployment.db_template}
                    Character set: {deployment.db_charset}
                    Additional data: {deployment.user_data}
                """,
            },
            'undeploy': {
                'summary': "Request for removing Oracle DB PaaS instance",
                'details': """
                    Customer name: {customer.name}
                    Project name: {project.project_group.name}
                    Environment name: {project.name}
                    Customer UUID: {customer.uuid.hex}
                    Project UUID: {project.project_group.uuid.hex}
                    Environment UUID: {project.uuid.hex}

                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}
                """,
            },
            'resize': {
                'summary': "Request for resizing Oracle DB PaaS instance",
                'details': """
                    Customer name: {customer.name}
                    Project name: {project.project_group.name}
                    Environment name: {project.name}
                    Customer UUID: {customer.uuid.hex}
                    Project UUID: {project.project_group.uuid.hex}
                    Environment UUID: {project.uuid.hex}

                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}

                    Hardware Configuration:
                    Flavor: {deployment.flavor_info}
                """,
            },
            'support': {
                'summary': "Custom support request",
                'details': """
                    Customer name: {customer.name}
                    Project name: {project.project_group.name}
                    Environment name: {project.name}
                    Customer UUID: {customer.uuid.hex}
                    Project UUID: {project.project_group.uuid.hex}
                    Environment UUID: {project.uuid.hex}

                    Oracle DB details:
                    Name: {deployment.name}
                    UUID: {deployment.uuid.hex}

                    {message}
                """,
            },
        }

    @staticmethod
    def django_app():
        return 'nodeconductor_paas_oracle'

    @staticmethod
    def rest_urls():
        from .urls import register_in
        return register_in
<|endoftext|>


<commit_before>#!/usr/bin/python3
# PYTHON_ARGCOMPLETE_OK

# Copyright 2019 Fiona Klute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import select
import socket
import subprocess
import sys
import traceback
import yaml

from http.client import HTTPConnection
from multiprocessing import Process
from time import sleep

class HTTPSubprocessConnection(HTTPConnection):
    def __init__(self, command, host, port=None,
                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                 blocksize=8192):
        super(HTTPSubprocessConnection, self).__init__(host, port, timeout,
                                                       source_address=None,
                                                       blocksize=blocksize)
        # "command" must be a list containing binary and command line
        # parameters
        self.command = command
        # This will be the subprocess reference when connected
        self._sproc = None
        # The subprocess return code is stored here on close()
        self.returncode = None
        # The set_tunnel method of the super class is not supported
        # (see exception doc)
        self.set_tunnel = None
        self._fproc = None

    @classmethod
    def _filter(cls, in_stream, out_stream):
        import fcntl
        import os
        # This filters out a log line about loading client
        # certificates that is mistakenly sent to stdout. My fix has
        # been merged, but buggy binaries will probably be around for
        # a while.
        # https://gitlab.com/gnutls/gnutls/merge_requests/1125
        cert_log = b'Processed 1 client X.509 certificates...\n'

        fd = in_stream.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
        poller = select.poll()
        poller.register(fd)

        run_loop = True
        while run_loop:
            # Critical: "event" is a bitwise OR of the POLL* constants
            for x, event in poller.poll():
                if event & select.POLLIN or event & select.POLLPRI:
                    data = in_stream.read()
                    if cert_log in data:
                        data = data.replace(cert_log, b'')
                    out_stream.send(data)
                if event & select.POLLHUP or event & select.POLLRDHUP:
                    # Stop the loop, but process any other events that
                    # might be in the list returned by poll() first.
                    run_loop = False

        in_stream.close()
        out_stream.close()

    def connect(self):
        s_local, s_remote = socket.socketpair(socket.AF_UNIX,
                                              socket.SOCK_STREAM)
        s_local.settimeout(self.timeout)

        # TODO: Maybe capture stderr?
        self._sproc = subprocess.Popen(self.command, stdout=subprocess.PIPE,
                                       stdin=s_remote, close_fds=True,
                                       bufsize=0)
        self._fproc = Process(target=HTTPSubprocessConnection._filter,
                              args=(self._sproc.stdout, s_remote))
        self._fproc.start()
        s_remote.close()
        self.sock = s_local

    def close(self):
        # close socket to subprocess for writing
        if self.sock:
            self.sock.shutdown(socket.SHUT_WR)

        # Wait for the process to stop, send SIGTERM/SIGKILL if
        # necessary
        try:
            self.returncode = self._sproc.wait(self.timeout)
        except subprocess.TimeoutExpired:
            try:
                self._sproc.terminate()
                self.returncode = self._sproc.wait(self.timeout)
            except subprocess.TimeoutExpired:
                self._sproc.kill()
                self.returncode = self._sproc.wait(self.timeout)

        # filter process receives HUP on pipe when the subprocess
        # terminates
        self._fproc.join()

        # close the connection in the super class, which also calls
        # self.sock.close()
        super().close()

class TestRequest(yaml.YAMLObject):
    yaml_tag = '!request'
    def __init__(self, path, method='GET', headers=dict(),
                 expect=dict(status=200)):
        self.method = method
        self.path = path
        self.headers = headers
        self.expect = expect

    def __repr__(self):
        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
                f'method={self.method!r}, headers={self.headers!r}, '
                f'expect={self.expect!r})')

    def _check_body(self, body):
        """
        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
        >>> r1._check_body('test\\n')
        >>> r1._check_body('xyz\\n')
        Traceback (most recent call last):
        ...
        https-test-client.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
        >>> r2._check_body('test\\n')
        >>> r2._check_body('est\\n')
        Traceback (most recent call last):
        ...
        https-test-client.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
        >>> r3._check_body('test\\n')
        """
        if 'exactly' in self.expect['body'] \
           and body != self.expect['body']['exactly']:
            raise TestExpectationFailed(
                f'Unexpected body: {body!r} != '
                f'{self.expect["body"]["exactly"]!r}')
        if 'contains' in self.expect['body']:
            if type(self.expect['body']['contains']) is str:
                self.expect['body']['contains'] = [
                    self.expect['body']['contains']]
            for s in self.expect['body']['contains']:
                if not s in body:
                    raise TestExpectationFailed(
                        f'Unexpected body: {body!r} does not contain '
                        f'{s!r}')

    def check_response(self, response, body):
        if self.expects_conn_reset():
            raise TestExpectationFailed(
                'Got a response, but connection should have failed!')
        if response.status != self.expect['status']:
            raise TestExpectationFailed(
                f'Unexpected status: {response.status} != '
                f'{self.expect["status"]}')
        if 'body' in self.expect:
            self._check_body(body)

    def expects_conn_reset(self):
        if 'reset' in self.expect:
            return self.expect['reset']
        return False

    @classmethod
    def _from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node)
        req = TestRequest(**fields)
        return req

class TestConnection(yaml.YAMLObject):
    yaml_tag = '!connection'

    def __init__(self, actions, gnutls_params=[], protocol='https'):
        self.gnutls_params = gnutls_params
        self.actions = actions
        self.protocol = protocol

    def __repr__(self):
        return (f'{self.__class__.__name__!s}'
                f'(gnutls_params={self.gnutls_params!r}, '
                f'actions={self.actions!r}, protocol={self.protocol!r})')

    @classmethod
    def _from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node)
        conn = TestConnection(**fields)
        return conn

# Override the default constructors. Pyyaml ignores default parameters
# otherwise.
yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)

class TestExpectationFailed(Exception):
    """Raise if a test failed. The constructor should be called with a
    string describing the problem."""
    pass

def format_response(resp, body):
    s = f'{resp.status} {resp.reason}\n'
    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
    s = s + '\n\n' + body
    return s

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(
        description='Send HTTP requests through gnutls-cli',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('host', nargs='?', help='Access the specified host',
                        default='localhost')
    parser.add_argument('--insecure', action='store_true',
                        help='do not validate the server certificate')
    parser.add_argument('-p', '--port', type=int,
                        help='Access the specified port', default='8000')
    parser.add_argument('--x509cafile', type=str,
                        help='Use the specified CA to validate the '
                        'server certificate')
    parser.add_argument('--test-config', type=argparse.FileType('r'),
                        help='load YAML test configuration')

    # enable bash completion if argcomplete is available
    try:
        import argcomplete
        argcomplete.autocomplete(parser)
    except ImportError:
        pass

    args = parser.parse_args()

    test_conn = None
    test_actions = None

    if args.test_config:
        config = yaml.load(args.test_config, Loader=yaml.Loader)
        if type(config) is TestConnection:
            test_conn = config
            print(test_conn)
            test_actions = test_conn.actions
    else:
        # simple default request
        test_actions = [TestRequest(path='/test.txt',
                                    expect={'status': 200, 'body': 'test\n'},
                                    method='GET')]

    # note: "--logfile" option requires GnuTLS version >= 3.6.7
    command = ['gnutls-cli', '--logfile=/dev/stderr']
    if args.insecure:
        command.append('--insecure')
    if args.x509cafile:
        command.append('--x509cafile')
        command.append(args.x509cafile)
    if test_conn != None:
        for s in test_conn.gnutls_params:
            command.append('--' + s)
    command = command + ['-p', str(args.port), args.host]

    conn = HTTPSubprocessConnection(command, args.host, port=args.port,
                                    timeout=6.0)

    try:
        for act in test_actions:
            if type(act) is TestRequest:
                try:
                    conn.request(act.method, act.path, headers=act.headers)
                    resp = conn.getresponse()
                except ConnectionResetError as err:
                    if act.expects_conn_reset():
                        print('connection reset as expected.')
                        break
                    else:
                        raise err
                body = resp.read().decode()
                print(format_response(resp, body))
                act.check_response(resp, body)
            else:
                raise TypeError(f'Unsupported action requested: {act!r}')
    finally:
        conn.close()
<commit_msg>https-test-client.py: Some more comments on the filter function<commit_after>#!/usr/bin/python3
# PYTHON_ARGCOMPLETE_OK

# Copyright 2019 Fiona Klute
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import select
import socket
import subprocess
import sys
import traceback
import yaml

from http.client import HTTPConnection
from multiprocessing import Process
from time import sleep

class HTTPSubprocessConnection(HTTPConnection):
    def __init__(self, command, host, port=None,
                 timeout=socket._GLOBAL_DEFAULT_TIMEOUT,
                 blocksize=8192):
        super(HTTPSubprocessConnection, self).__init__(host, port, timeout,
                                                       source_address=None,
                                                       blocksize=blocksize)
        # "command" must be a list containing binary and command line
        # parameters
        self.command = command
        # This will be the subprocess reference when connected
        self._sproc = None
        # The subprocess return code is stored here on close()
        self.returncode = None
        # The set_tunnel method of the super class is not supported
        # (see exception doc)
        self.set_tunnel = None
        self._fproc = None

    @classmethod
    def _filter(cls, in_stream, out_stream):
        import fcntl
        import os
        # This filters out a log line about loading client
        # certificates that is mistakenly sent to stdout. My fix has
        # been merged, but buggy binaries will probably be around for
        # a while.
        # https://gitlab.com/gnutls/gnutls/merge_requests/1125
        cert_log = b'Processed 1 client X.509 certificates...\n'

        # Set the input to non-blocking mode
        fd = in_stream.fileno()
        fl = fcntl.fcntl(fd, fcntl.F_GETFL)
        fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)

        # The poll object allows waiting for events on non-blocking IO
        # channels.
        poller = select.poll()
        poller.register(fd)

        run_loop = True
        while run_loop:
            # The returned tuples are file descriptor and event, but
            # we're only listening on one stream anyway, so we don't
            # need to check it here.
            for x, event in poller.poll():
                # Critical: "event" is a bitwise OR of the POLL* constants
                if event & select.POLLIN or event & select.POLLPRI:
                    data = in_stream.read()
                    if cert_log in data:
                        data = data.replace(cert_log, b'')
                    out_stream.send(data)
                if event & select.POLLHUP or event & select.POLLRDHUP:
                    # Stop the loop, but process any other events that
                    # might be in the list returned by poll() first.
                    run_loop = False

        in_stream.close()
        out_stream.close()

    def connect(self):
        s_local, s_remote = socket.socketpair(socket.AF_UNIX,
                                              socket.SOCK_STREAM)
        s_local.settimeout(self.timeout)

        # TODO: Maybe capture stderr?
        self._sproc = subprocess.Popen(self.command, stdout=subprocess.PIPE,
                                       stdin=s_remote, close_fds=True,
                                       bufsize=0)
        self._fproc = Process(target=HTTPSubprocessConnection._filter,
                              args=(self._sproc.stdout, s_remote))
        self._fproc.start()
        s_remote.close()
        self.sock = s_local

    def close(self):
        # close socket to subprocess for writing
        if self.sock:
            self.sock.shutdown(socket.SHUT_WR)

        # Wait for the process to stop, send SIGTERM/SIGKILL if
        # necessary
        try:
            self.returncode = self._sproc.wait(self.timeout)
        except subprocess.TimeoutExpired:
            try:
                self._sproc.terminate()
                self.returncode = self._sproc.wait(self.timeout)
            except subprocess.TimeoutExpired:
                self._sproc.kill()
                self.returncode = self._sproc.wait(self.timeout)

        # filter process receives HUP on pipe when the subprocess
        # terminates
        self._fproc.join()

        # close the connection in the super class, which also calls
        # self.sock.close()
        super().close()

class TestRequest(yaml.YAMLObject):
    yaml_tag = '!request'
    def __init__(self, path, method='GET', headers=dict(),
                 expect=dict(status=200)):
        self.method = method
        self.path = path
        self.headers = headers
        self.expect = expect

    def __repr__(self):
        return (f'{self.__class__.__name__!s}(path={self.path!r}, '
                f'method={self.method!r}, headers={self.headers!r}, '
                f'expect={self.expect!r})')

    def _check_body(self, body):
        """
        >>> r1 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'exactly': 'test\\n'}})
        >>> r1._check_body('test\\n')
        >>> r1._check_body('xyz\\n')
        Traceback (most recent call last):
        ...
        https-test-client.TestExpectationFailed: Unexpected body: 'xyz\\n' != 'test\\n'
        >>> r2 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': ['tes', 'est']}})
        >>> r2._check_body('test\\n')
        >>> r2._check_body('est\\n')
        Traceback (most recent call last):
        ...
        https-test-client.TestExpectationFailed: Unexpected body: 'est\\n' does not contain 'tes'
        >>> r3 = TestRequest(path='/test.txt', method='GET', headers={}, expect={'status': 200, 'body': {'contains': 'test'}})
        >>> r3._check_body('test\\n')
        """
        if 'exactly' in self.expect['body'] \
           and body != self.expect['body']['exactly']:
            raise TestExpectationFailed(
                f'Unexpected body: {body!r} != '
                f'{self.expect["body"]["exactly"]!r}')
        if 'contains' in self.expect['body']:
            if type(self.expect['body']['contains']) is str:
                self.expect['body']['contains'] = [
                    self.expect['body']['contains']]
            for s in self.expect['body']['contains']:
                if not s in body:
                    raise TestExpectationFailed(
                        f'Unexpected body: {body!r} does not contain '
                        f'{s!r}')

    def check_response(self, response, body):
        if self.expects_conn_reset():
            raise TestExpectationFailed(
                'Got a response, but connection should have failed!')
        if response.status != self.expect['status']:
            raise TestExpectationFailed(
                f'Unexpected status: {response.status} != '
                f'{self.expect["status"]}')
        if 'body' in self.expect:
            self._check_body(body)

    def expects_conn_reset(self):
        if 'reset' in self.expect:
            return self.expect['reset']
        return False

    @classmethod
    def _from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node)
        req = TestRequest(**fields)
        return req

class TestConnection(yaml.YAMLObject):
    yaml_tag = '!connection'

    def __init__(self, actions, gnutls_params=[], protocol='https'):
        self.gnutls_params = gnutls_params
        self.actions = actions
        self.protocol = protocol

    def __repr__(self):
        return (f'{self.__class__.__name__!s}'
                f'(gnutls_params={self.gnutls_params!r}, '
                f'actions={self.actions!r}, protocol={self.protocol!r})')

    @classmethod
    def _from_yaml(cls, loader, node):
        fields = loader.construct_mapping(node)
        conn = TestConnection(**fields)
        return conn

# Override the default constructors. Pyyaml ignores default parameters
# otherwise.
yaml.add_constructor('!request', TestRequest._from_yaml, yaml.Loader)
yaml.add_constructor('!connection', TestConnection._from_yaml, yaml.Loader)

class TestExpectationFailed(Exception):
    """Raise if a test failed. The constructor should be called with a
    string describing the problem."""
    pass

def format_response(resp, body):
    s = f'{resp.status} {resp.reason}\n'
    s = s + '\n'.join(f'{name}: {value}' for name, value in resp.getheaders())
    s = s + '\n\n' + body
    return s

if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(
        description='Send HTTP requests through gnutls-cli',
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('host', nargs='?', help='Access the specified host',
                        default='localhost')
    parser.add_argument('--insecure', action='store_true',
                        help='do not validate the server certificate')
    parser.add_argument('-p', '--port', type=int,
                        help='Access the specified port', default='8000')
    parser.add_argument('--x509cafile', type=str,
                        help='Use the specified CA to validate the '
                        'server certificate')
    parser.add_argument('--test-config', type=argparse.FileType('r'),
                        help='load YAML test configuration')

    # enable bash completion if argcomplete is available
    try:
        import argcomplete
        argcomplete.autocomplete(parser)
    except ImportError:
        pass

    args = parser.parse_args()

    test_conn = None
    test_actions = None

    if args.test_config:
        config = yaml.load(args.test_config, Loader=yaml.Loader)
        if type(config) is TestConnection:
            test_conn = config
            print(test_conn)
            test_actions = test_conn.actions
    else:
        # simple default request
        test_actions = [TestRequest(path='/test.txt',
                                    expect={'status': 200, 'body': 'test\n'},
                                    method='GET')]

    # note: "--logfile" option requires GnuTLS version >= 3.6.7
    command = ['gnutls-cli', '--logfile=/dev/stderr']
    if args.insecure:
        command.append('--insecure')
    if args.x509cafile:
        command.append('--x509cafile')
        command.append(args.x509cafile)
    if test_conn != None:
        for s in test_conn.gnutls_params:
            command.append('--' + s)
    command = command + ['-p', str(args.port), args.host]

    conn = HTTPSubprocessConnection(command, args.host, port=args.port,
                                    timeout=6.0)

    try:
        for act in test_actions:
            if type(act) is TestRequest:
                try:
                    conn.request(act.method, act.path, headers=act.headers)
                    resp = conn.getresponse()
                except ConnectionResetError as err:
                    if act.expects_conn_reset():
                        print('connection reset as expected.')
                        break
                    else:
                        raise err
                body = resp.read().decode()
                print(format_response(resp, body))
                act.check_response(resp, body)
            else:
                raise TypeError(f'Unsupported action requested: {act!r}')
    finally:
        conn.close()
<|endoftext|>


<commit_before># coding=utf-8

#
# Load train/test datasets as Pandas DataFrames
df_train = pd.read_csv('Data/train.csv', index_col='PassengerId')
df_test = pd.read_csv('Data/test.csv', index_col='PassengerId')
df = pd.concat([df_train, df_test], axis=0)

<commit_msg>Add New Feature (family_size)<commit_after># coding=utf-8

#
# Load train/test datasets as Pandas DataFrames
df_train = pd.read_csv('Data/train.csv', index_col='PassengerId')
df_test = pd.read_csv('Data/test.csv', index_col='PassengerId')
df = pd.concat([df_train, df_test], axis=0)

# Add New Feature (family_size)
# `family_size` is calculated using the following features: `sibsp` & `parch`
df.family_size = df.sibsp + df.parch + 1

<|endoftext|>


<commit_before># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path
from typing import Any, Dict, List

import requests
from pandas import DataFrame, concat, isna

from lib.case_line import convert_cases_to_time_series
from lib.cast import safe_int_cast, numeric_code_as_string
from lib.pipeline import DataSource
from lib.time import datetime_isoformat
from lib.utils import table_rename

_IBGE_STATES = {
    # Norte
    "RO": 11,
    "AC": 12,
    "AM": 13,
    "RR": 14,
    "PA": 15,
    "AP": 16,
    "TO": 17,
    # Nordeste
    "MA": 21,
    "PI": 22,
    "CE": 23,
    "RN": 24,
    "PB": 25,
    "PE": 26,
    "AL": 27,
    "SE": 28,
    "BA": 29,
    # Sudeste
    "MG": 31,
    "ES": 32,
    "RJ": 33,
    "SP": 35,
    # Sul
    "PR": 41,
    "SC": 42,
    "RS": 43,
    # Centro-Oeste
    "MS": 50,
    "MT": 51,
    "GO": 52,
    "DF": 53,
}

class BrazilMunicipalitiesDataSource(DataSource):
    def fetch(
        self, output_folder: Path, cache: Dict[str, str], fetch_opts: List[Dict[str, Any]]
    ) -> Dict[str, str]:
        # Get the URL from a fake browser request
        url = requests.get(
            "https://xx9p7hp1p7.execute-api.us-east-1.amazonaws.com/prod/PortalGeral",
            headers={
                "Accept": "application/json, text/plain, */*",
                "Accept-Language": "en-GB,en;q=0.5",
                "X-Parse-Application-Id": "unAFkcaNDeXajurGB7LChj8SgQYS2ptm",
                "Origin": "https://covid.saude.gov.br",
                "Connection": "keep-alive",
                "Referer": "https://covid.saude.gov.br/",
                "Pragma": "no-cache",
                "Cache-Control": "no-cache",
                "TE": "Trailers",
            },
        ).json()["results"][0]["arquivo"]["url"]

        # Pass the actual URL down to fetch it
        return super().fetch(output_folder, cache, [{"url": url}])

    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:

        data = table_rename(
            dataframes[0],
            {
                "data": "date",
                "estado": "subregion1_code",
                "codmun": "subregion2_code",
                "municipio": "subregion2_name",
                "casosNovos": "new_confirmed",
                "obitosNovos": "new_deceased",
                "casosAcumulado": "total_confirmed",
                "obitosAcumulado": "total_deceased",
                "Recuperadosnovos": "total_recovered",
            },
            drop=True,
        )

        # Convert date to ISO format
        data["date"] = data["date"].astype(str)

        # Parse region codes as strings
        data["subregion2_code"] = data["subregion2_code"].apply(
            lambda x: numeric_code_as_string(x, 6)
        )

        # Country-level data has null state
        data["key"] = None
        country_mask = data["subregion1_code"].isna()
        data.loc[country_mask, "key"] = "BR"

        # State-level data has null municipality
        state_mask = data["subregion2_code"].isna()
        data.loc[~country_mask & state_mask, "key"] = "BR_" + data["subregion1_code"]

        # We can derive the key from subregion1 + subregion2
        data.loc[~country_mask & ~state_mask, "key"] = (
            "BR_" + data["subregion1_code"] + "_" + data["subregion2_code"]
        )

        # Drop bogus data
        data = data[data["subregion2_code"].str.slice(-4) != "0000"]

        return data

_column_adapter = {
    "sexo": "sex",
    "idade": "age",
    "municipioIBGE": "subregion2_code",
    "dataTeste": "date_new_tested",
    "dataInicioSintomas": "_date_onset",
    "estadoIBGE": "_state_code",
    "evolucaoCaso": "_prognosis",
    "dataEncerramento": "_date_update",
    "resultadoTeste": "_test_result",
    "classificacaoFinal": "_classification",
}

class BrazilStratifiedDataSource(DataSource):
    def fetch(
        self, output_folder: Path, cache: Dict[str, str], fetch_opts: List[Dict[str, Any]]
    ) -> Dict[str, str]:
        # The source URL is a template which we must format for the requested state
        parse_opts = self.config["parse"]
        fetch_opts = [
            {**opts, "url": opts["url"].format(parse_opts["subregion1_code"].lower())}
            for opts in fetch_opts
        ]
        return super().fetch(output_folder, cache, fetch_opts)

    def parse(self, sources: Dict[str, str], aux: Dict[str, DataFrame], **parse_opts) -> DataFrame:
        # Manipulate the parse options here because we have access to the columns adapter
        parse_opts = {**parse_opts, "error_bad_lines": False, "usecols": _column_adapter.keys()}
        return super().parse(sources, aux, **parse_opts)

    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:

        cases = table_rename(dataframes[0], _column_adapter, drop=True)

        # Keep only cases for a single state
        subregion1_code = parse_opts["subregion1_code"]
        cases = cases[cases["_state_code"].apply(safe_int_cast) == _IBGE_STATES[subregion1_code]]

        # Confirmed cases are only those with a confirmed positive test result
        cases["date_new_confirmed"] = None
        confirmed_mask = cases["_test_result"] == "Positivo"
        cases.loc[confirmed_mask, "date_new_confirmed"] = cases.loc[
            confirmed_mask, "date_new_tested"
        ]

        # Deceased cases have a specific label and the date is the "closing" date
        cases["date_new_deceased"] = None
        deceased_mask = cases["_prognosis"] == "Óbito"
        cases.loc[confirmed_mask, "date_new_deceased"] = cases.loc[deceased_mask, "_date_update"]

        # Recovered cases have a specific label and the date is the "closing" date
        cases["date_new_recovered"] = None
        recovered_mask = cases["_prognosis"] == "Cured"
        cases.loc[confirmed_mask, "date_new_recovered"] = cases.loc[recovered_mask, "_date_update"]

        # Drop columns which we have no use for
        cases = cases[[col for col in cases.columns if not col.startswith("_")]]

        # Subregion code comes from the parsing parameters
        cases["subregion1_code"] = subregion1_code

        # Make sure our region code is of type str
        cases["subregion2_code"] = cases["subregion2_code"].apply(safe_int_cast)
        # The last digit of the region code is actually not necessary
        cases["subregion2_code"] = cases["subregion2_code"].apply(
            lambda x: None if isna(x) else str(int(x))[:-1]
        )

        # Drop bogus records from the data
        cases = cases[~cases["subregion1_code"].isna() & (cases["subregion1_code"] != "")]
        cases = cases[~cases["subregion2_code"].isna() & (cases["subregion2_code"] != "")]

        # We can derive the key from subregion1 + subregion2
        cases["key"] = "BR_" + cases["subregion1_code"] + "_" + cases["subregion2_code"]

        # Convert ages to int, and translate sex (no "other" sex/gender reported)
        cases["age"] = cases["age"].apply(safe_int_cast)
        cases["sex"] = (
            cases["sex"].str.lower().apply({"masculino": "male", "femenino": "female"}.get)
        )

        # Convert to time series format
        data = convert_cases_to_time_series(cases, index_columns=["key"])

        # Convert date to ISO format
        data["date"] = data["date"].apply(lambda x: datetime_isoformat(x, "%Y-%m-%dT%H:%M:%S.%fZ"))

        # Aggregate for the whole state
        state = data.drop(columns=["key"]).groupby(["date", "age", "sex"]).sum().reset_index()
        state["key"] = f"BR_{subregion1_code}"

        return concat([data, state])
<commit_msg>Fix BR female counts (#108)<commit_after># Copyright 2020 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from pathlib import Path
from typing import Any, Dict, List

import requests
from pandas import DataFrame, concat, isna

from lib.case_line import convert_cases_to_time_series
from lib.cast import safe_int_cast, numeric_code_as_string
from lib.pipeline import DataSource
from lib.time import datetime_isoformat
from lib.utils import table_rename

_IBGE_STATES = {
    # Norte
    "RO": 11,
    "AC": 12,
    "AM": 13,
    "RR": 14,
    "PA": 15,
    "AP": 16,
    "TO": 17,
    # Nordeste
    "MA": 21,
    "PI": 22,
    "CE": 23,
    "RN": 24,
    "PB": 25,
    "PE": 26,
    "AL": 27,
    "SE": 28,
    "BA": 29,
    # Sudeste
    "MG": 31,
    "ES": 32,
    "RJ": 33,
    "SP": 35,
    # Sul
    "PR": 41,
    "SC": 42,
    "RS": 43,
    # Centro-Oeste
    "MS": 50,
    "MT": 51,
    "GO": 52,
    "DF": 53,
}

class BrazilMunicipalitiesDataSource(DataSource):
    def fetch(
        self, output_folder: Path, cache: Dict[str, str], fetch_opts: List[Dict[str, Any]]
    ) -> Dict[str, str]:
        # Get the URL from a fake browser request
        url = requests.get(
            "https://xx9p7hp1p7.execute-api.us-east-1.amazonaws.com/prod/PortalGeral",
            headers={
                "Accept": "application/json, text/plain, */*",
                "Accept-Language": "en-GB,en;q=0.5",
                "X-Parse-Application-Id": "unAFkcaNDeXajurGB7LChj8SgQYS2ptm",
                "Origin": "https://covid.saude.gov.br",
                "Connection": "keep-alive",
                "Referer": "https://covid.saude.gov.br/",
                "Pragma": "no-cache",
                "Cache-Control": "no-cache",
                "TE": "Trailers",
            },
        ).json()["results"][0]["arquivo"]["url"]

        # Pass the actual URL down to fetch it
        return super().fetch(output_folder, cache, [{"url": url}])

    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:

        data = table_rename(
            dataframes[0],
            {
                "data": "date",
                "estado": "subregion1_code",
                "codmun": "subregion2_code",
                "municipio": "subregion2_name",
                "casosNovos": "new_confirmed",
                "obitosNovos": "new_deceased",
                "casosAcumulado": "total_confirmed",
                "obitosAcumulado": "total_deceased",
                "Recuperadosnovos": "total_recovered",
            },
            drop=True,
        )

        # Convert date to ISO format
        data["date"] = data["date"].astype(str)

        # Parse region codes as strings
        data["subregion2_code"] = data["subregion2_code"].apply(
            lambda x: numeric_code_as_string(x, 6)
        )

        # Country-level data has null state
        data["key"] = None
        country_mask = data["subregion1_code"].isna()
        data.loc[country_mask, "key"] = "BR"

        # State-level data has null municipality
        state_mask = data["subregion2_code"].isna()
        data.loc[~country_mask & state_mask, "key"] = "BR_" + data["subregion1_code"]

        # We can derive the key from subregion1 + subregion2
        data.loc[~country_mask & ~state_mask, "key"] = (
            "BR_" + data["subregion1_code"] + "_" + data["subregion2_code"]
        )

        # Drop bogus data
        data = data[data["subregion2_code"].str.slice(-4) != "0000"]

        return data

_column_adapter = {
    "sexo": "sex",
    "idade": "age",
    "municipioIBGE": "subregion2_code",
    "dataTeste": "date_new_tested",
    "dataInicioSintomas": "_date_onset",
    "estadoIBGE": "_state_code",
    "evolucaoCaso": "_prognosis",
    "dataEncerramento": "_date_update",
    "resultadoTeste": "_test_result",
    "classificacaoFinal": "_classification",
}

class BrazilStratifiedDataSource(DataSource):
    def fetch(
        self, output_folder: Path, cache: Dict[str, str], fetch_opts: List[Dict[str, Any]]
    ) -> Dict[str, str]:
        # The source URL is a template which we must format for the requested state
        parse_opts = self.config["parse"]
        fetch_opts = [
            {**opts, "url": opts["url"].format(parse_opts["subregion1_code"].lower())}
            for opts in fetch_opts
        ]
        return super().fetch(output_folder, cache, fetch_opts)

    def parse(self, sources: Dict[str, str], aux: Dict[str, DataFrame], **parse_opts) -> DataFrame:
        # Manipulate the parse options here because we have access to the columns adapter
        parse_opts = {**parse_opts, "error_bad_lines": False, "usecols": _column_adapter.keys()}
        return super().parse(sources, aux, **parse_opts)

    def parse_dataframes(
        self, dataframes: Dict[str, DataFrame], aux: Dict[str, DataFrame], **parse_opts
    ) -> DataFrame:

        cases = table_rename(dataframes[0], _column_adapter, drop=True)

        # Keep only cases for a single state
        subregion1_code = parse_opts["subregion1_code"]
        cases = cases[cases["_state_code"].apply(safe_int_cast) == _IBGE_STATES[subregion1_code]]

        # Confirmed cases are only those with a confirmed positive test result
        cases["date_new_confirmed"] = None
        confirmed_mask = cases["_test_result"] == "Positivo"
        cases.loc[confirmed_mask, "date_new_confirmed"] = cases.loc[
            confirmed_mask, "date_new_tested"
        ]

        # Deceased cases have a specific label and the date is the "closing" date
        cases["date_new_deceased"] = None
        deceased_mask = cases["_prognosis"] == "Óbito"
        cases.loc[confirmed_mask, "date_new_deceased"] = cases.loc[deceased_mask, "_date_update"]

        # Recovered cases have a specific label and the date is the "closing" date
        cases["date_new_recovered"] = None
        recovered_mask = cases["_prognosis"] == "Cured"
        cases.loc[confirmed_mask, "date_new_recovered"] = cases.loc[recovered_mask, "_date_update"]

        # Drop columns which we have no use for
        cases = cases[[col for col in cases.columns if not col.startswith("_")]]

        # Subregion code comes from the parsing parameters
        cases["subregion1_code"] = subregion1_code

        # Make sure our region code is of type str
        cases["subregion2_code"] = cases["subregion2_code"].apply(safe_int_cast)
        # The last digit of the region code is actually not necessary
        cases["subregion2_code"] = cases["subregion2_code"].apply(
            lambda x: None if isna(x) else str(int(x))[:-1]
        )

        # Drop bogus records from the data
        cases = cases[~cases["subregion1_code"].isna() & (cases["subregion1_code"] != "")]
        cases = cases[~cases["subregion2_code"].isna() & (cases["subregion2_code"] != "")]

        # We can derive the key from subregion1 + subregion2
        cases["key"] = "BR_" + cases["subregion1_code"] + "_" + cases["subregion2_code"]

        # Convert ages to int, and translate sex (no "other" sex/gender reported)
        cases["age"] = cases["age"].apply(safe_int_cast)
        cases["sex"] = (
            cases["sex"]
            .str.lower()
            .apply({"masculino": "male", "feminino": "female", "indefinido": None}.get)
        )

        # Convert to time series format
        data = convert_cases_to_time_series(cases, index_columns=["key"])

        # Convert date to ISO format
        data["date"] = data["date"].apply(lambda x: datetime_isoformat(x, "%Y-%m-%dT%H:%M:%S.%fZ"))

        # Aggregate for the whole state
        state = data.drop(columns=["key"]).groupby(["date", "age", "sex"]).sum().reset_index()
        state["key"] = f"BR_{subregion1_code}"

        return concat([data, state])
<|endoftext|>


<commit_before>#
# ovirt-hosted-engine-setup -- ovirt hosted engine setup
# Copyright (C) 2013-2017 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""
bridge configuration plugin.
"""

import ethtool
import gettext
import socket

from otopi import plugin
from otopi import util

from ovirt_setup_lib import hostname as osetuphostname

from ovirt_hosted_engine_setup import ansible_utils
from ovirt_hosted_engine_setup import constants as ohostedcons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-hosted-engine-setup')

@util.export
class Plugin(plugin.PluginBase):
    """
    bridge configuration plugin.
    """

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
    )
    def _init(self):
        self.environment.setdefault(
            ohostedcons.NetworkEnv.BRIDGE_IF,
            None
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.BRIDGE_NAME,
            ohostedcons.Defaults.DEFAULT_BRIDGE_NAME
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.REFUSE_DEPLOYING_WITH_NM,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.FQDN_REVERSE_VALIDATION,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.ALLOW_INVALID_BOND_MODES,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.HOST_NAME,
            None
        )
        self.environment.setdefault(
            ohostedcons.EngineEnv.APP_HOST_NAME,
            None
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_SETUP,
    )
    def _setup(self):
        self._hostname_helper = osetuphostname.Hostname(plugin=self)

    @plugin.event(
        stage=plugin.Stages.STAGE_PROGRAMS,
        condition=lambda self: self.environment[
            ohostedcons.NetworkEnv.REFUSE_DEPLOYING_WITH_NM
        ]
    )
    def _check_NM(self):
        nmstatus = self.services.status('NetworkManager')
        self.logger.debug('NetworkManager: {status}'.format(
            status=nmstatus,
        ))
        if nmstatus:
            raise RuntimeError(_(
                'hosted-engine cannot be deployed while NetworkManager is '
                'running, please stop and disable it before proceeding'
            ))

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        name=ohostedcons.Stages.BRIDGE_DETECTED,
        after=(
            ohostedcons.Stages.REQUIRE_ANSWER_FILE,
            ohostedcons.Stages.DIALOG_TITLES_S_NETWORK,
        ),
        before=(
            ohostedcons.Stages.DIALOG_TITLES_E_NETWORK,
        ),
    )
    def _detect_bridges(self):
        if (
            self.environment[ohostedcons.NetworkEnv.BRIDGE_NAME] in
            ethtool.get_devices()
        ):
            self.logger.info(
                _(
                    'Bridge {bridge} already created'
                ).format(
                    bridge=self.environment[ohostedcons.NetworkEnv.BRIDGE_NAME]
                )
            )

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        after=(
            ohostedcons.Stages.DIALOG_TITLES_S_NETWORK,
            ohostedcons.Stages.BRIDGE_DETECTED,
        ),
        before=(
            ohostedcons.Stages.DIALOG_TITLES_E_NETWORK,
        ),
    )
    def _customization(self):
        validValues = []
        ah = ansible_utils.AnsibleHelper(
            tags=ohostedcons.Const.HE_TAG_NETWORK_INTERFACES,
            extra_vars={'he_just_collect_network_interfaces': True},
        )
        r = ah.run()
        self.logger.debug(r)
        try:
            validValues = r[
                'otopi_host_net'
            ]['ansible_facts']['otopi_host_net']
        except KeyError:
            raise RuntimeError(_(
                'No suitable network interfaces were found'
            ))
        if not validValues:
            raise RuntimeError(_('A Network interface is required'))
        interactive = self.environment[
            ohostedcons.NetworkEnv.BRIDGE_IF
        ] is None
        if interactive:
            default = self._get_active_interface(validValues)
            self.environment[
                ohostedcons.NetworkEnv.BRIDGE_IF
            ] = self.dialog.queryString(
                name='ovehosted_bridge_if',
                note=_(
                    'Please indicate a nic to set '
                    '{bridge} bridge on: (@VALUES@) [@DEFAULT@]: '
                ).format(
                    bridge=self.environment[
                        ohostedcons.NetworkEnv.BRIDGE_NAME
                    ]
                ),
                prompt=True,
                caseSensitive=True,
                default=default,
                validValues=validValues,
            )

    def _get_active_interface(self, valid_interfaces):
        for iface in valid_interfaces:
            try:
                if (
                    socket.getfqdn(ethtool.get_ipaddr(iface)) ==
                    socket.gethostname()
                ):
                    return iface
            except IOError:
                pass
        return valid_interfaces[0]

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
        name=ohostedcons.Stages.GOT_HOSTNAME_FIRST_HOST,
    )
    def _get_hostname_from_bridge_if(self):
        if not self.environment[
            ohostedcons.NetworkEnv.HOST_NAME
        ]:
            self.environment[
                ohostedcons.NetworkEnv.HOST_NAME
            ] = socket.gethostname()
        if not self.environment[
            ohostedcons.EngineEnv.APP_HOST_NAME
        ]:
            self.environment[
                ohostedcons.EngineEnv.APP_HOST_NAME
            ] = socket.gethostname()

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
        after=(
            ohostedcons.Stages.GOT_HOSTNAME_FIRST_HOST,
        ),
    )
    def _validate_hostname_first_host(self):
        self._hostname_helper.getHostname(
            envkey=ohostedcons.NetworkEnv.HOST_NAME,
            whichhost='HostedEngine',
            supply_default=False,
            validate_syntax=True,
            system=True,
            dns=True,
            local_non_loopback=True,
            reverse_dns=self.environment[
                ohostedcons.NetworkEnv.FQDN_REVERSE_VALIDATION
            ],
            not_local=False,
            allow_empty=False,
        )

# vim: expandtab tabstop=4 shiftwidth=4
<commit_msg>network: let the user interactively validate host address<commit_after>#
# ovirt-hosted-engine-setup -- ovirt hosted engine setup
# Copyright (C) 2013-2017 Red Hat, Inc.
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#

"""
bridge configuration plugin.
"""

import ethtool
import gettext
import socket

from otopi import plugin
from otopi import util

from ovirt_setup_lib import hostname as osetuphostname

from ovirt_hosted_engine_setup import ansible_utils
from ovirt_hosted_engine_setup import constants as ohostedcons

def _(m):
    return gettext.dgettext(message=m, domain='ovirt-hosted-engine-setup')

@util.export
class Plugin(plugin.PluginBase):
    """
    bridge configuration plugin.
    """

    def __init__(self, context):
        super(Plugin, self).__init__(context=context)

    @plugin.event(
        stage=plugin.Stages.STAGE_INIT,
    )
    def _init(self):
        self.environment.setdefault(
            ohostedcons.NetworkEnv.BRIDGE_IF,
            None
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.BRIDGE_NAME,
            ohostedcons.Defaults.DEFAULT_BRIDGE_NAME
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.REFUSE_DEPLOYING_WITH_NM,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.FQDN_REVERSE_VALIDATION,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.ALLOW_INVALID_BOND_MODES,
            False
        )
        self.environment.setdefault(
            ohostedcons.NetworkEnv.HOST_NAME,
            None
        )
        self.environment.setdefault(
            ohostedcons.EngineEnv.APP_HOST_NAME,
            None
        )

    @plugin.event(
        stage=plugin.Stages.STAGE_SETUP,
    )
    def _setup(self):
        self._hostname_helper = osetuphostname.Hostname(plugin=self)

    @plugin.event(
        stage=plugin.Stages.STAGE_PROGRAMS,
        condition=lambda self: self.environment[
            ohostedcons.NetworkEnv.REFUSE_DEPLOYING_WITH_NM
        ]
    )
    def _check_NM(self):
        nmstatus = self.services.status('NetworkManager')
        self.logger.debug('NetworkManager: {status}'.format(
            status=nmstatus,
        ))
        if nmstatus:
            raise RuntimeError(_(
                'hosted-engine cannot be deployed while NetworkManager is '
                'running, please stop and disable it before proceeding'
            ))

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        name=ohostedcons.Stages.BRIDGE_DETECTED,
        after=(
            ohostedcons.Stages.REQUIRE_ANSWER_FILE,
            ohostedcons.Stages.DIALOG_TITLES_S_NETWORK,
        ),
        before=(
            ohostedcons.Stages.DIALOG_TITLES_E_NETWORK,
        ),
    )
    def _detect_bridges(self):
        if (
            self.environment[ohostedcons.NetworkEnv.BRIDGE_NAME] in
            ethtool.get_devices()
        ):
            self.logger.info(
                _(
                    'Bridge {bridge} already created'
                ).format(
                    bridge=self.environment[ohostedcons.NetworkEnv.BRIDGE_NAME]
                )
            )

    @plugin.event(
        stage=plugin.Stages.STAGE_CUSTOMIZATION,
        after=(
            ohostedcons.Stages.DIALOG_TITLES_S_NETWORK,
            ohostedcons.Stages.BRIDGE_DETECTED,
        ),
        before=(
            ohostedcons.Stages.DIALOG_TITLES_E_NETWORK,
        ),
    )
    def _customization(self):
        validValues = []
        ah = ansible_utils.AnsibleHelper(
            tags=ohostedcons.Const.HE_TAG_NETWORK_INTERFACES,
            extra_vars={'he_just_collect_network_interfaces': True},
        )
        r = ah.run()
        self.logger.debug(r)
        try:
            validValues = r[
                'otopi_host_net'
            ]['ansible_facts']['otopi_host_net']
        except KeyError:
            raise RuntimeError(_(
                'No suitable network interfaces were found'
            ))
        if not validValues:
            raise RuntimeError(_('A Network interface is required'))
        interactive = self.environment[
            ohostedcons.NetworkEnv.BRIDGE_IF
        ] is None
        if interactive:
            default = self._get_active_interface(validValues)
            self.environment[
                ohostedcons.NetworkEnv.BRIDGE_IF
            ] = self.dialog.queryString(
                name='ovehosted_bridge_if',
                note=_(
                    'Please indicate a nic to set '
                    '{bridge} bridge on: (@VALUES@) [@DEFAULT@]: '
                ).format(
                    bridge=self.environment[
                        ohostedcons.NetworkEnv.BRIDGE_NAME
                    ]
                ),
                prompt=True,
                caseSensitive=True,
                default=default,
                validValues=validValues,
            )

    def _get_active_interface(self, valid_interfaces):
        for iface in valid_interfaces:
            try:
                if (
                    socket.getfqdn(ethtool.get_ipaddr(iface)) ==
                    socket.gethostname()
                ):
                    return iface
            except IOError:
                pass
        return valid_interfaces[0]

    @plugin.event(
        stage=plugin.Stages.STAGE_VALIDATION,
    )
    def _validate_hostname_first_host(self):
        self._hostname_helper.getHostname(
            envkey=ohostedcons.NetworkEnv.HOST_NAME,
            whichhost='first HE',
            prompttext=(
                'Please provide the hostname of this host '
                'on the management network'
            ),
            supply_default=True,
            validate_syntax=True,
            system=True,
            dns=True,
            local_non_loopback=True,
            reverse_dns=self.environment[
                ohostedcons.NetworkEnv.FQDN_REVERSE_VALIDATION
            ],
            not_local=False,
            allow_empty=False,
        )
        if not self.environment[
            ohostedcons.EngineEnv.APP_HOST_NAME
        ]:
            self.environment[
                ohostedcons.EngineEnv.APP_HOST_NAME
            ] = self.environment[
                ohostedcons.NetworkEnv.HOST_NAME
            ]

# vim: expandtab tabstop=4 shiftwidth=4
<|endoftext|>


<commit_before># Copyright 2016 by Raytheon BBN Technologies Corp.  All Rights Reserved.

# QGL2 clean versions for T1T2.py

from qgl2.qgl2 import qgl2decl, qbit
from qgl2.basic_sequences.helpers import create_cal_seqs
from qgl2.util import init
from qgl2.qgl1 import QubitFactory, X, Id, MEAS, U90, X90
import numpy as np
from numpy import pi

@qgl2decl
def doInversionRecovery():
    # FIXME: No args possible yet
    q = QubitFactory('q1')
    delays = np.linspace(0, 5e-6, 11)
    calRepeats = 2
    for d in delays:
        init(q)
        X(q)
        Id(q, length=d)
        MEAS(q)

    # Tack on calibration
    create_cal_seqs((q,), calRepeats)

@qgl2decl
def doRamsey():
    # FIXME: No args possible yet: TPPIFreq, pulseSpacings, calRepeats
    q = QubitFactory('q1')
    pulseSpacings=np.arange(100e-9, 10e-6, 100e-9)
    TPPIFreq=1e6 # 0
    calRepeats = 2

    # Create the phases for the TPPI
    phases = 2*pi*TPPIFreq*pulseSpacings

    # Create the basic Ramsey sequence
    # FIXME: QGL2 doesn't deal well with this call to zip: make it a list
    for d,phase in list(zip(pulseSpacings, phases)):
        init(q)
        X90(q)
        Id(q, length=d)
        U90(q, phase=phase)
        MEAS(q)

    # Tack on calibration
    create_cal_seqs((q,), calRepeats)

@qgl2decl
def doRamsey_nolist():
    # FIXME: No args possible yet: TPPIFreq, pulseSpacings, calRepeats
    q = QubitFactory('q1')
    pulseSpacings=np.arange(100e-9, 10e-6, 100e-9)
    TPPIFreq=1e6 # 0
    calRepeats = 2

    # Create the phases for the TPPI
    phases = 2*pi*TPPIFreq*pulseSpacings

    # Create the basic Ramsey sequence
    # FIXME: QGL2 doesn't deal well with this call to zip: make it a list
    for d,phase in zip(pulseSpacings, phases):
        init(q)
        X90(q)
        Id(q, length=d)
        U90(q, phase=phase)
        MEAS(q)

    # Tack on calibration
    create_cal_seqs((q,), calRepeats)
<commit_msg>Revert "add a test that we can can loop over generators"<commit_after># Copyright 2016 by Raytheon BBN Technologies Corp.  All Rights Reserved.

# QGL2 clean versions for T1T2.py

from qgl2.qgl2 import qgl2decl, qbit
from qgl2.basic_sequences.helpers import create_cal_seqs
from qgl2.util import init
from qgl2.qgl1 import QubitFactory, X, Id, MEAS, U90, X90
import numpy as np
from numpy import pi

@qgl2decl
def doInversionRecovery():
    # FIXME: No args possible yet
    q = QubitFactory('q1')
    delays = np.linspace(0, 5e-6, 11)
    calRepeats = 2
    for d in delays:
        init(q)
        X(q)
        Id(q, length=d)
        MEAS(q)

    # Tack on calibration
    create_cal_seqs((q,), calRepeats)

@qgl2decl
def doRamsey():
    # FIXME: No args possible yet: TPPIFreq, pulseSpacings, calRepeats
    q = QubitFactory('q1')
    pulseSpacings=np.arange(100e-9, 10e-6, 100e-9)
    TPPIFreq=1e6 # 0
    calRepeats = 2

    # Create the phases for the TPPI
    phases = 2*pi*TPPIFreq*pulseSpacings

    # Create the basic Ramsey sequence
    # FIXME: QGL2 doesn't deal well with this call to zip: make it a list
    for d,phase in list(zip(pulseSpacings, phases)):
        init(q)
        X90(q)
        Id(q, length=d)
        U90(q, phase=phase)
        MEAS(q)

    # Tack on calibration
    create_cal_seqs((q,), calRepeats)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
"""
run_tests.py

Created by Scott on 2012-12-26.
Copyright (c) 2012 Scott Rice. All rights reserved.
"""

import sys, os, inspect
from test import run_tests

# The code below is to allow test cases to import the class they are testing by
# using the syntax 'import ice.******'.
#
# This code was taken from a StackOverflow answer by sorin. The internet is
# down right now but I will link to it when I get the chance.
#
# Get a reference to the current directory, without using __file__, which fails
# in certain situations based on how you call the script in Windows
cmd_folder = os.path.realpath(os.path.abspath(os.path.split(inspect.getfile(inspect.currentframe()))[0]))
if cmd_folder not in sys.path:
    sys.path.insert(0,cmd_folder)

run_tests()<commit_msg>Properly attributed some code I got from StackOverflow<commit_after>#!/usr/bin/env python
# encoding: utf-8
"""
run_tests.py

Created by Scott on 2012-12-26.
Copyright (c) 2012 Scott Rice. All rights reserved.
"""

import sys, os, inspect
from test import run_tests

# The code below is to allow test cases to import the class they are testing by
# using the syntax 'import ice.******'.
#
# This code was taken from a StackOverflow answer by sorin. Thanks bud!
# http://stackoverflow.com/questions/279237/python-import-a-module-from-a-folder
#
# Get a reference to the current directory, without using __file__, which fails
# in certain situations based on how you call the script in Windows
cmd_folder = os.path.realpath(os.path.abspath(os.path.split(inspect.getfile(inspect.currentframe()))[0]))
if cmd_folder not in sys.path:
    sys.path.insert(0,cmd_folder)

run_tests()<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8
import sys
import os
from django.core.management import call_command, execute_manager, execute_from_command_line
from django.conf import settings

def use_exec(pdir):
    """
    instead, employ the technique used by django.core.management.execute_manager()
    """
    #import settings
    print "Installing media"
    # RUNNING MANAGEMENT COMMANDS CAUSES WIERD SETTINGS STUFF IN TESTS
    # ALL these cause subsequent manage.py test commands to fail:
    #
    # command.execute()
    # execute_manager(settings,['manage.py','install_media'])
    # execute_from_command_line(['manage.py','install_media'])
    # call_command('install_media')
    #
    # So we have to call install_media from the shell
    manage_path = os.path.join(pdir,'manage.py')
    res = os.popen("python %s install_media" % manage_path).read()
    print res

    print "Executing tests from command line"
    #execute_from_command_line(['manage.py','test','array','--failfast','-v','2'])
    execute_from_command_line(['manage.py','test'])

hdir = os.path.dirname(os.path.abspath(__file__))
pdir = os.path.join(hdir,'example_projects/test_project')
spdir = os.path.join(hdir,'example_projects')
sys.path.insert(0, pdir)
sys.path.insert(0, spdir)
#os.chdir(pdir)
os.environ['DJANGO_SETTINGS_MODULE'] = 'test_project.settings'

settings.TEST_RUNNER='xmlrunner.extra.djangotestrunner.run_tests'
settings.TEST_OUTPUT_DESCRIPTIONS=True
settings.DEBUG=True
settings.POSTGIS_TEMPLATE='template1'

use_exec(pdir)
<commit_msg>Backed out changeset 14756d0c88e5<commit_after>#!/usr/bin/env python
# encoding: utf-8
import sys
import os
from django.core.management import call_command, execute_manager, execute_from_command_line
from django.conf import settings

def use_exec(pdir):
    """
    instead, employ the technique used by django.core.management.execute_manager()
    """
    #import settings
    print "Installing media"
    from lingcod.common.management.commands.install_media import Command as InstallMediaCommand
    command = InstallMediaCommand()
    # RUNNING COMMANDS CAUSES WIERD SETTINGS STUFF
    # The solution is a horrible, horrible hack
    # We're forced to reimplement the install_media command
    # ALL these cause subsequent management commands to fail:
    #
    # command.execute()
    # execute_manager(settings,['manage.py','install_media'])
    # execute_from_command_line(['manage.py','install_media'])
    # call_command('install_media')
    #
    command.dry_run = False
    command.media_root = settings.MEDIA_ROOT
    command.force_compress = True

    lingcod_media_dir = command.get_lingcod_dir()
    project_media_dir = command.get_project_dir()

    if os.path.abspath(os.path.realpath(lingcod_media_dir)) == os.path.abspath(os.path.realpath(command.media_root)) or \
        os.path.abspath(os.path.realpath(project_media_dir)) == os.path.abspath(os.path.realpath(command.media_root)):
        raise Exception("Your MEDIA_ROOT setting has to be a directory other than your lingcod or project media folder!")

    command.copy_media_to_root(lingcod_media_dir)
    command.copy_media_to_root(project_media_dir)
    # Cant call synccompress so can't do command.compile_media()
    manage_path = os.path.join(pdir,'manage.py')
    res = os.popen("python %s synccompress" % manage_path).read()
    print res

    print "Executing from command line"
    #execute_from_command_line(['manage.py','test','array','--failfast','-v','2'])
    execute_from_command_line(['manage.py','test'])

hdir = os.path.dirname(os.path.abspath(__file__))
pdir = os.path.join(hdir,'example_projects/test_project')
spdir = os.path.join(hdir,'example_projects')
sys.path.insert(0, pdir)
sys.path.insert(0, spdir)
#os.chdir(pdir)
os.environ['DJANGO_SETTINGS_MODULE'] = 'test_project.settings'

settings.TEST_RUNNER='xmlrunner.extra.djangotestrunner.run_tests'
settings.TEST_OUTPUT_DESCRIPTIONS=True
settings.DEBUG=True
settings.POSTGIS_TEMPLATE='template1'

use_exec(pdir)
<|endoftext|>