<commit_before># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# nova documentation build configuration file, created by
# sphinx-quickstart on Sat May  1 15:17:47 2010.
#
# This file is execfile()d with the current directory set to
# its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import os
import subprocess
import sys
import warnings

# TODO(Graham Hayes): Remove the following block of code when os-api-ref is
# using openstackdocstheme

import os_api_ref

if getattr(os_api_ref, 'THEME', 'olsosphinx') == 'openstackdocstheme':
    # We are on the new version with openstackdocstheme support

    extensions = [
        'os_api_ref',
    ]

    import openstackdocstheme  # noqa

    html_theme = 'openstackdocs'
    html_theme_path = [openstackdocstheme.get_html_theme_path()]
    html_theme_options = {
        "sidebar_mode": "toc",
    }

else:
    # We are on the old version without openstackdocstheme support

    extensions = [
        'os_api_ref',
        'oslosphinx',
    ]

# End temporary block

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../../'))
sys.path.insert(0, os.path.abspath('../'))
sys.path.insert(0, os.path.abspath('./'))

# -- General configuration ----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#
# source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Compute API Reference'
copyright = u'2010-present, OpenStack Foundation'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
from ceilometer.version import version_info as ceilometer_version
# The full version, including alpha/beta/rc tags.
release = ceilometer_version.version_string_with_vcs()
# The short X.Y version.
version = ceilometer_version.canonical_version_string()

# Config logABug feature
giturl = (
    u'http://git.openstack.org/cgit/openstack/ceilometer/tree/api-ref/source')
# source tree
# html_context allows us to pass arbitrary values into the html template
html_context = {'bug_tag': 'api-ref',
                'giturl': giturl,
                'bug_project': 'ceilometer'}

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# The reST default role (used for this markup: `text`) to use
# for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# -- Options for man page output ----------------------------------------------

# Grouping the document tree for man pages.
# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'

# -- Options for HTML output --------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme_path = ["."]
# html_theme = '_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'
git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
           "-n1"]
try:
    html_last_updated_fmt = subprocess.Popen(
        git_cmd, stdout=subprocess.PIPE).communicate()[0].decode()
except Exception:
    warnings.warn('Cannot get last updated time from git repository. '
                  'Not setting "html_last_updated_fmt".')

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_use_modindex = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'novadoc'

# -- Options for LaTeX output -------------------------------------------------

# The paper size ('letter' or 'a4').
# latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
# latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
    ('index', 'CeilometerReleaseNotes.tex',
     u'Ceilometer Release Notes Documentation',
     u'Ceilometer Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# Additional stuff for the LaTeX preamble.
# latex_preamble = ''

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_use_modindex = True
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'ceilometerreleasenotes',
     u'Ceilometer Release Notes Documentation', [u'Ceilometer Developers'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'CeilometerReleaseNotes',
     u'Ceilometer Release Notes Documentation',
     u'Ceilometer Developers', 'CeilometerReleaseNotes',
     'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False
<commit_msg>[api-ref] Remove temporary block in conf.py<commit_after># -*- coding: utf-8 -*-
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# nova documentation build configuration file, created by
# sphinx-quickstart on Sat May  1 15:17:47 2010.
#
# This file is execfile()d with the current directory set to
# its containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

import os
import subprocess
import sys
import warnings

import openstackdocstheme

html_theme = 'openstackdocs'
html_theme_path = [openstackdocstheme.get_html_theme_path()]
html_theme_options = {
    "sidebar_mode": "toc",
}

extensions = [
    'os_api_ref',
]

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
sys.path.insert(0, os.path.abspath('../../'))
sys.path.insert(0, os.path.abspath('../'))
sys.path.insert(0, os.path.abspath('./'))

# -- General configuration ----------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom ones.

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
#
# source_encoding = 'utf-8'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'Compute API Reference'
copyright = u'2010-present, OpenStack Foundation'

# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
from ceilometer.version import version_info as ceilometer_version
# The full version, including alpha/beta/rc tags.
release = ceilometer_version.version_string_with_vcs()
# The short X.Y version.
version = ceilometer_version.canonical_version_string()

# Config logABug feature
giturl = (
    u'http://git.openstack.org/cgit/openstack/ceilometer/tree/api-ref/source')
# source tree
# html_context allows us to pass arbitrary values into the html template
html_context = {'bug_tag': 'api-ref',
                'giturl': giturl,
                'bug_project': 'ceilometer'}

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# The reST default role (used for this markup: `text`) to use
# for all documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
add_module_names = False

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'

# -- Options for man page output ----------------------------------------------

# Grouping the document tree for man pages.
# List of tuples 'sourcefile', 'target', u'title', u'Authors name', 'manual'

# -- Options for HTML output --------------------------------------------------

# The theme to use for HTML and HTML Help pages.  Major themes that come with
# Sphinx are currently 'default' and 'sphinxdoc'.
# html_theme_path = ["."]
# html_theme = '_theme'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
# html_static_path = ['_static']

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'
git_cmd = ["git", "log", "--pretty=format:'%ad, commit %h'", "--date=local",
           "-n1"]
try:
    html_last_updated_fmt = subprocess.Popen(
        git_cmd, stdout=subprocess.PIPE).communicate()[0].decode()
except Exception:
    warnings.warn('Cannot get last updated time from git repository. '
                  'Not setting "html_last_updated_fmt".')

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_use_modindex = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# If nonempty, this is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = ''

# Output file base name for HTML help builder.
htmlhelp_basename = 'novadoc'

# -- Options for LaTeX output -------------------------------------------------

# The paper size ('letter' or 'a4').
# latex_paper_size = 'letter'

# The font size ('10pt', '11pt' or '12pt').
# latex_font_size = '10pt'

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title, author, documentclass
# [howto/manual]).
latex_documents = [
    ('index', 'CeilometerReleaseNotes.tex',
     u'Ceilometer Release Notes Documentation',
     u'Ceilometer Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# Additional stuff for the LaTeX preamble.
# latex_preamble = ''

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_use_modindex = True
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'ceilometerreleasenotes',
     u'Ceilometer Release Notes Documentation', [u'Ceilometer Developers'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'CeilometerReleaseNotes',
     u'Ceilometer Release Notes Documentation',
     u'Ceilometer Developers', 'CeilometerReleaseNotes',
     'One line description of project.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
__version__ = "0.18.1"
<commit_msg>Version v0.18.2 ready for tag<commit_after># -*- coding: utf-8 -*-
__version__ = "0.18.2"
<|endoftext|>


<commit_before>from app import app
from flask import jsonify, request, send_file
from werkzeug.exceptions import BadRequest, NotFound

from datetime import timedelta, date

from newman_es.es_series import get_entity_histogram
from param_utils import parseParamDatetime, parseParamEmailAddress, parseParamEntity, parseParamTextQuery, parseParamEncrypted, parseParamCommunityIds
#
from newman_es.es_queries import _build_email_query
from newman_es.es_query_utils import _query_emails
from newman_es.es_search import _build_graph_for_emails, _query_email_attachments

from newman_es.es_email import get_top_attachment_types
from newman_es.es_series import get_email_activity, get_total_attachment_activity, get_emailer_attachment_activity, attachment_histogram
from newman_es.es_topic import get_categories, get_dynamic_clusters
from newman_es.es_numeric_aggregations import get_top_phone_numbers

from datetime import datetime

def _interval_bounds(str_date1, str_date2, max_minor_ticks=200):
    # dates must be in format 'yyyy-mm-dd'
    date1 = datetime.strptime(str_date1, '%Y-%m-%d')
    date2 = datetime.strptime(str_date2, '%Y-%m-%d')
    delta_days = (date2 - date1).days
    if(delta_days < max_minor_ticks):
        return "day"
    elif(delta_days / 7 < max_minor_ticks):
        return 'week'
    elif(delta_days / 30 < max_minor_ticks):
        return 'month'
    elif(delta_days / (365.0/4.0) < max_minor_ticks):
        return 'quarter'
    else:
        return 'year'

#GET <host>:<port>:/entity/entity?entities.entity_person=mike,joe&entities.entity_location=paris,los angeles
@app.route('/entity/entity')
def get_graph_for_entity():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args);
    entity_dict = parseParamEntity(request.args)
    qs = parseParamTextQuery(request.args)

    # TODO set from UI
    size = size if size >500 else 2500

    query = _build_email_query(email_addrs=email_address_list, qs=qs, entity=entity_dict, date_bounds=(start_datetime, end_datetime))
    app.logger.debug("query: %s" % (query))

    results = _query_emails(data_set_id, query, size)
    graph = _build_graph_for_emails(data_set_id, results["hits"])

    # Get attachments for community
    query = _build_email_query(email_addrs=email_address_list, qs=qs, entity=entity_dict, date_bounds=(start_datetime, end_datetime), attachments_only=True)
    app.logger.debug("attachment-query: %s" % (query))
    attachments = _query_email_attachments(data_set_id, query, size)
    graph["attachments"] = attachments["hits"]
    graph["attachments_total"] = attachments["attachments_total"]
    graph["query_hits"] = results["total"]

    return jsonify(graph)

#GET /top/<count>
@app.route('/entity/top/<int:top_count>')
def get_top_entities(top_count=20):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args);

    # TODO
    qs = parseParamTextQuery(request.args)

    if not email_address_list :
        # TODO qs not being evaluated in inner filter called by this method
        entities = get_entity_histogram(data_set_id, "emails", qs=qs, date_bounds=(start_datetime, end_datetime))[:top_count]
        result = {"entities" :
                  [
                   [
                    str(i),
                    entity ["type"],
                    entity ["key"],
                    entity ["doc_count"]
                   ] for i,entity in enumerate(entities)
                  ]
                 }

    else:
        # TODO qs not being evaluated in inner filter called by this method
        entities = get_entity_histogram(data_set_id, "emails", email_address_list, qs=qs, date_bounds=(start_datetime, end_datetime))[:top_count]
        result = {"entities" :
                  [
                   [
                    str(i),
                    entity ["type"],
                    entity ["key"],
                    entity ["doc_count"]
                   ] for i,entity in enumerate(entities)
                  ]
                 }

    return jsonify(result)

#GET attachment/types/<file_type>?data_set_id=<data_set>&start_datetime=<yyyy-mm-dd>&end_datetime=<yyyy-mm-dd>&size=<top_count>
@app.route('/attachment/types')
def getAttachFileType():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    top_count = int(size)

    encrypted = parseParamEncrypted(request.args)

    file_types = get_top_attachment_types(data_set_id, date_bounds=(start_datetime, end_datetime), encrypted=encrypted, num_top_attachments=top_count)[:top_count]

    result = {
              "account_id" : data_set_id,
              "data_set_id" : data_set_id,
              "account_start_datetime" : start_datetime,
              "account_end_datetime" : end_datetime,
              "types" : file_types
             }

    return jsonify(result)

#
#
def dateRange(start_datetime, end_datetime):
    for n in range(int ((end_datetime - start_datetime).days)):
        yield start_datetime + timedelta(n)

#GET /account/<account_type>
#GET /account/<account_type>?user0@gbc.com=1&user1@abc.com=1&...&data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
@app.route('/activity/account/all')
def getAccountActivity():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    email_address_list = parseParamEmailAddress(request.args);

    # TODO set from UI based on rendering area
    max_minor_ticks=800

    interval = _interval_bounds(start_datetime, end_datetime, max_minor_ticks)
    # interval = 'week'

    if not email_address_list :
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : data_set_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_email_activity(data_set_id, data_set_id, date_bounds=(start_datetime, end_datetime), interval=interval)
                   }
                  ]
                 }
    else:
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : account_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_email_activity(data_set_id, data_set_id, account_id, date_bounds=(start_datetime, end_datetime), interval=interval)
                   } for account_id in email_address_list
                  ]
                 }

    return jsonify(result)

#GET /attach/<attach_type>?data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
#GET /attach/<attach_type>?user0@gbc.com=1&user1@abc.com=1&...&data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
@app.route('/activity/attach/all')
def getAttachCount(*args, **kwargs):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    email_address_list = parseParamEmailAddress(request.args);

    if not email_address_list :
        activity = get_total_attachment_activity(data_set_id, data_set_id, query_function=attachment_histogram, sender_email_addr="", start=start_datetime, end=end_datetime, interval="week")
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : data_set_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : activity
                   }
                  ]
                 }

    else:
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : account_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_emailer_attachment_activity(data_set_id, account_id, (start_datetime, end_datetime), interval="week")
                   } for account_id in email_address_list
                  ]
                 }

    return jsonify(result)

# GET /topic/<querystr>?data_set_id=<>&start_datetime=<>&end_datetime=<>&size=<>&algorithm=<>&analysis_field=<list of fields from ES>
# analysis_field should be a field name in elasticsearch where the data to cluster is located.  This is optional as it defaults to "_source.body" but can be set to "_source.attachments.content" or "_all" or anything valid
@app.route('/topic/topic')
def get_topics_by_query():
    algorithm = request.args.get('algorithm', 'lingo')
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args)

    community = parseParamCommunityIds(request.args)
    qs=parseParamTextQuery(request.args)

    # TODO set from UI
    analysis_field = request.args.get("analysis_field","_source.body")
    ndocs = request.args.get("ndocs",0)
    doc_ids= bool(request.args.get("doc_ids",False))
    qs_hint= request.args.get("qs_hint",'')

    # TODO set from UI
    n_clusters_returned = 50

    clusters, total_docs = get_dynamic_clusters(data_set_id, "emails", email_addrs=email_address_list, qs=qs, qs_hint=qs_hint, date_bounds=(start_datetime, end_datetime), community=community, cluster_fields=[analysis_field], cluster_title_fields=[analysis_field], algorithm=algorithm, max_doc_pool_size=size, docs_return_size=ndocs, doc_ids=doc_ids)

    return jsonify({"topics" : clusters[:n_clusters_returned], "total_docs" : total_docs})

#GET /category/<category>
# returns topic in sorted order by the idx
@app.route('/topic/category/all/<int:num_topics>')
def topic_list(num_topics):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    return jsonify(get_categories(data_set_id))

#GET <host>:<port>/profile/top_phone_numbers?qs="<query_string>"
@app.route('/profile/top_phone_numbers')
def top_phone_numbers():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    qs = parseParamTextQuery(request.args)

    return jsonify(get_top_phone_numbers(data_set_id, email_address='', qs=qs, date_bounds=(start_datetime, end_datetime), size=size))

<commit_msg>adjusted the default max_minor_ticks for the datetime series;<commit_after>from app import app
from flask import jsonify, request, send_file
from werkzeug.exceptions import BadRequest, NotFound

from datetime import timedelta, date

from newman_es.es_series import get_entity_histogram
from param_utils import parseParamDatetime, parseParamEmailAddress, parseParamEntity, parseParamTextQuery, parseParamEncrypted, parseParamCommunityIds
#
from newman_es.es_queries import _build_email_query
from newman_es.es_query_utils import _query_emails
from newman_es.es_search import _build_graph_for_emails, _query_email_attachments

from newman_es.es_email import get_top_attachment_types
from newman_es.es_series import get_email_activity, get_total_attachment_activity, get_emailer_attachment_activity, attachment_histogram
from newman_es.es_topic import get_categories, get_dynamic_clusters
from newman_es.es_numeric_aggregations import get_top_phone_numbers

from datetime import datetime

def _interval_bounds(str_date1, str_date2, max_minor_ticks=200):
    # dates must be in format 'yyyy-mm-dd'
    date1 = datetime.strptime(str_date1, '%Y-%m-%d')
    date2 = datetime.strptime(str_date2, '%Y-%m-%d')
    delta_days = (date2 - date1).days
    if(delta_days < max_minor_ticks):
        return "day"
    elif(delta_days / 7 < max_minor_ticks):
        return 'week'
    elif(delta_days / 30 < max_minor_ticks):
        return 'month'
    elif(delta_days / (365.0/4.0) < max_minor_ticks):
        return 'quarter'
    else:
        return 'year'

#GET <host>:<port>:/entity/entity?entities.entity_person=mike,joe&entities.entity_location=paris,los angeles
@app.route('/entity/entity')
def get_graph_for_entity():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args);
    entity_dict = parseParamEntity(request.args)
    qs = parseParamTextQuery(request.args)

    # TODO set from UI
    size = size if size >500 else 2500

    query = _build_email_query(email_addrs=email_address_list, qs=qs, entity=entity_dict, date_bounds=(start_datetime, end_datetime))
    app.logger.debug("query: %s" % (query))

    results = _query_emails(data_set_id, query, size)
    graph = _build_graph_for_emails(data_set_id, results["hits"])

    # Get attachments for community
    query = _build_email_query(email_addrs=email_address_list, qs=qs, entity=entity_dict, date_bounds=(start_datetime, end_datetime), attachments_only=True)
    app.logger.debug("attachment-query: %s" % (query))
    attachments = _query_email_attachments(data_set_id, query, size)
    graph["attachments"] = attachments["hits"]
    graph["attachments_total"] = attachments["attachments_total"]
    graph["query_hits"] = results["total"]

    return jsonify(graph)

#GET /top/<count>
@app.route('/entity/top/<int:top_count>')
def get_top_entities(top_count=20):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args);

    # TODO
    qs = parseParamTextQuery(request.args)

    if not email_address_list :
        # TODO qs not being evaluated in inner filter called by this method
        entities = get_entity_histogram(data_set_id, "emails", qs=qs, date_bounds=(start_datetime, end_datetime))[:top_count]
        result = {"entities" :
                  [
                   [
                    str(i),
                    entity ["type"],
                    entity ["key"],
                    entity ["doc_count"]
                   ] for i,entity in enumerate(entities)
                  ]
                 }

    else:
        # TODO qs not being evaluated in inner filter called by this method
        entities = get_entity_histogram(data_set_id, "emails", email_address_list, qs=qs, date_bounds=(start_datetime, end_datetime))[:top_count]
        result = {"entities" :
                  [
                   [
                    str(i),
                    entity ["type"],
                    entity ["key"],
                    entity ["doc_count"]
                   ] for i,entity in enumerate(entities)
                  ]
                 }

    return jsonify(result)

#GET attachment/types/<file_type>?data_set_id=<data_set>&start_datetime=<yyyy-mm-dd>&end_datetime=<yyyy-mm-dd>&size=<top_count>
@app.route('/attachment/types')
def getAttachFileType():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    top_count = int(size)

    encrypted = parseParamEncrypted(request.args)

    file_types = get_top_attachment_types(data_set_id, date_bounds=(start_datetime, end_datetime), encrypted=encrypted, num_top_attachments=top_count)[:top_count]

    result = {
              "account_id" : data_set_id,
              "data_set_id" : data_set_id,
              "account_start_datetime" : start_datetime,
              "account_end_datetime" : end_datetime,
              "types" : file_types
             }

    return jsonify(result)

#
#
def dateRange(start_datetime, end_datetime):
    for n in range(int ((end_datetime - start_datetime).days)):
        yield start_datetime + timedelta(n)

#GET /account/<account_type>
#GET /account/<account_type>?user0@gbc.com=1&user1@abc.com=1&...&data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
@app.route('/activity/account/all')
def getAccountActivity():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    email_address_list = parseParamEmailAddress(request.args);

    # TODO set from UI based on rendering area
    max_minor_ticks=812

    interval = _interval_bounds(start_datetime, end_datetime, max_minor_ticks)
    # interval = 'week'

    if not email_address_list :
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : data_set_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_email_activity(data_set_id, data_set_id, date_bounds=(start_datetime, end_datetime), interval=interval)
                   }
                  ]
                 }
    else:
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : account_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_email_activity(data_set_id, data_set_id, account_id, date_bounds=(start_datetime, end_datetime), interval=interval)
                   } for account_id in email_address_list
                  ]
                 }

    return jsonify(result)

#GET /attach/<attach_type>?data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
#GET /attach/<attach_type>?user0@gbc.com=1&user1@abc.com=1&...&data_set_id=<id>&start_datetime=<datetime>&end_datetime=<datetime>
@app.route('/activity/attach/all')
def getAttachCount(*args, **kwargs):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    email_address_list = parseParamEmailAddress(request.args);

    if not email_address_list :
        activity = get_total_attachment_activity(data_set_id, data_set_id, query_function=attachment_histogram, sender_email_addr="", start=start_datetime, end=end_datetime, interval="week")
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : data_set_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : activity
                   }
                  ]
                 }

    else:
        result = {"account_activity_list" :
                  [
                   {
                    "account_id" : account_id,
                    "data_set_id" : data_set_id,
                    "account_start_datetime" : start_datetime,
                    "account_end_datetime" : end_datetime,
                    "activities" : get_emailer_attachment_activity(data_set_id, account_id, (start_datetime, end_datetime), interval="week")
                   } for account_id in email_address_list
                  ]
                 }

    return jsonify(result)

# GET /topic/<querystr>?data_set_id=<>&start_datetime=<>&end_datetime=<>&size=<>&algorithm=<>&analysis_field=<list of fields from ES>
# analysis_field should be a field name in elasticsearch where the data to cluster is located.  This is optional as it defaults to "_source.body" but can be set to "_source.attachments.content" or "_all" or anything valid
@app.route('/topic/topic')
def get_topics_by_query():
    algorithm = request.args.get('algorithm', 'lingo')
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    email_address_list = parseParamEmailAddress(request.args)

    community = parseParamCommunityIds(request.args)
    qs=parseParamTextQuery(request.args)

    # TODO set from UI
    analysis_field = request.args.get("analysis_field","_source.body")
    ndocs = request.args.get("ndocs",0)
    doc_ids= bool(request.args.get("doc_ids",False))
    qs_hint= request.args.get("qs_hint",'')

    # TODO set from UI
    n_clusters_returned = 50

    clusters, total_docs = get_dynamic_clusters(data_set_id, "emails", email_addrs=email_address_list, qs=qs, qs_hint=qs_hint, date_bounds=(start_datetime, end_datetime), community=community, cluster_fields=[analysis_field], cluster_title_fields=[analysis_field], algorithm=algorithm, max_doc_pool_size=size, docs_return_size=ndocs, doc_ids=doc_ids)

    return jsonify({"topics" : clusters[:n_clusters_returned], "total_docs" : total_docs})

#GET /category/<category>
# returns topic in sorted order by the idx
@app.route('/topic/category/all/<int:num_topics>')
def topic_list(num_topics):
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)
    return jsonify(get_categories(data_set_id))

#GET <host>:<port>/profile/top_phone_numbers?qs="<query_string>"
@app.route('/profile/top_phone_numbers')
def top_phone_numbers():
    data_set_id, start_datetime, end_datetime, size, _from = parseParamDatetime(request.args)

    qs = parseParamTextQuery(request.args)

    return jsonify(get_top_phone_numbers(data_set_id, email_address='', qs=qs, date_bounds=(start_datetime, end_datetime), size=size))

<|endoftext|>


<commit_before>#!/usr/bin/env python
"""Tests for the Model Migration feature"""

from __future__ import print_function

import argparse
from contextlib import contextmanager
import logging
import os
from subprocess import CalledProcessError
import sys
from time import sleep

from assess_user_grant_revoke import User
from deploy_stack import BootstrapManager
from jujucharm import local_charm_path
from utility import (
    JujuAssertionError,
    add_basic_testing_arguments,
    configure_logging,
    temp_dir,
    until_timeout,
)

__metaclass__ = type

log = logging.getLogger("assess_model_migration")

def assess_model_migration(bs1, bs2, upload_tools):
    with bs1.booted_context(upload_tools):
        bs1.client.enable_feature('migration')

        bs2.client.env.juju_home = bs1.client.env.juju_home
        with bs2.existing_booted_context(upload_tools):
            ensure_able_to_migrate_model_between_controllers(
                bs1, bs2, upload_tools)

            with temp_dir() as temp:
                ensure_migrating_with_insufficient_user_permissions_fails(
                    bs1, bs2, upload_tools, temp)
                ensure_migrating_with_superuser_user_permissions_succeeds(
                    bs1, bs2, upload_tools, temp)

            ensure_migration_rolls_back_on_failure(bs1, bs2, upload_tools)

def parse_args(argv):
    """Parse all arguments."""
    parser = argparse.ArgumentParser(
        description="Test model migration feature"
    )
    add_basic_testing_arguments(parser)
    return parser.parse_args(argv)

def get_bootstrap_managers(args):
    """Create 2 bootstrap managers from the provided args.

    Need to make a couple of elements uniqe (e.g. environment name) so we can
    have 2 bootstrapped at the same time.

    """
    bs_1 = BootstrapManager.from_args(args)
    bs_2 = BootstrapManager.from_args(args)

    # Give the second a separate/unique name.
    bs_2.temp_env_name = '{}-b'.format(bs_1.temp_env_name)

    bs_1.log_dir = _new_log_dir(args.logs, 'a')
    bs_2.log_dir = _new_log_dir(args.logs, 'b')

    return bs_1, bs_2

def _new_log_dir(log_dir, post_fix):
    new_log_dir = os.path.join(log_dir, 'env-{}'.format(post_fix))
    os.mkdir(new_log_dir)
    return new_log_dir

def wait_for_model(client, model_name, timeout=60):
    """Wait for a given `timeout` for a model of `model_name` to appear within
    `client`.

    Defaults to 10 seconds timeout.
    :raises AssertionError: If the named model does not appear in the specified
      timeout.

    """
    with client.check_timeouts():
        with client.ignore_soft_deadline():
            for _ in until_timeout(timeout):
                models = client.get_models()
                if model_name in [m['name'] for m in models['models']]:
                    return
                sleep(1)
            raise JujuAssertionError(
                'Model \'{}\' failed to appear after {} seconds'.format(
                    model_name, timeout
                ))

def test_deployed_mongo_is_up(client):
    """Ensure the mongo service is running as expected."""
    try:
        output = client.get_juju_output(
            'run', '--unit', 'mongodb/0', 'mongo --eval "db.getMongo()"')
        if 'connecting to: test' in output:
            return
    except CalledProcessError as e:
        # Pass through to assertion error
        log.error('Mongodb check command failed: {}'.format(e))
    raise AssertionError('Mongo db is not in an expected state.')

def ensure_able_to_migrate_model_between_controllers(
        source_environ, dest_environ, upload_tools):
    """Test simple migration of a model to another controller.

    Ensure that migration a model that has an application deployed upon it is
    able to continue it's operation after the migration process.

    Given 2 bootstrapped environments:
      - Deploy an application
        - ensure it's operating as expected
      - Migrate that model to the other environment
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional
      - Migrate the model back to the original environment
        - Note: Test for lp:1607457
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional

    """
    bundle = 'mongodb'
    application = 'mongodb'

    log.info('Deploying charm')
    # Don't move the default model so we can reuse it in later tests.
    test_model = source_environ.client.add_model(
        source_environ.client.env.clone('example-model'))
    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    log.info('Initiating migration process')

    migration_target_client = migrate_model_to_controller(
        test_model, dest_environ.client)

    migration_target_client.wait_for_workloads()
    test_deployed_mongo_is_up(migration_target_client)
    ensure_model_is_functional(migration_target_client, application)

    migration_target_client.remove_service(application)

def migrate_model_to_controller(source_client, dest_client):
    source_client.controller_juju(
        'migrate',
        (source_client.env.environment,
         dest_client.env.controller.name))

    migration_target_client = dest_client.clone(
        dest_client.env.clone(
            source_client.env.environment))

    wait_for_model(
        migration_target_client, source_client.env.environment)

    migration_target_client.wait_for_started()

    return migration_target_client

def ensure_model_is_functional(client, application):
    """Ensures that the migrated model is functional

    Add unit to application to ensure the model is contactable and working.
    Ensure that added unit is created on a new machine (check for bug
    LP:1607599)

    """
    client.juju('add-unit', (application,))
    client.wait_for_started()

    assert_units_on_different_machines(client, application)

def assert_units_on_different_machines(client, application):
    status = client.get_status()
    unit_machines = [u[1]['machine'] for u in status.iter_units()]

    raise_if_shared_machines(unit_machines)

def raise_if_shared_machines(unit_machines):
    """Raise an exception if `unit_machines` contain double ups of machine ids.

    A unique list of machine ids will be equal in length to the set of those
    machine ids.

    :raises ValueError: if an empty list is passed in.
    :raises JujuAssertionError: if any double-ups of machine ids are detected.

    """
    if not unit_machines:
        raise ValueError('Cannot share 0 machines. Empty list provided.')
    if len(unit_machines) != len(set(unit_machines)):
        raise JujuAssertionError('Appliction units reside on the same machine')

def ensure_migration_rolls_back_on_failure(source_bs, dest_bs, upload_tools):
    """Must successfully roll back migration when migration fails.

    If the target controller becomes unavailable for the migration to complete
    the migration must roll back.

    """
    source_client = source_bs.client
    dest_client = dest_bs.client

    bundle = 'mongodb'
    application = 'mongodb'

    test_model = source_client.add_model(
        source_client.env.clone('example-model'))

    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    source_client.controller_juju(
        'migrate',
        (source_client.env.environment,
         dest_client.env.controller.name))
    # Immediately disrupt the migration.
    with disable_apiserver(dest_client.get_controller_client()):
        # Wait for model to be back and working on the original controller.
        wait_for_model(test_model, source_client.env.environment)

@contextmanager
def disable_apiserver(admin_client):
    try:
        admin_client.juju('ssh', ('0', 'service jujud-machine-0 stop'))
        yield
    finally:
        admin_client.juju('ssh', ('0', 'service jujud-machine-0 start'))

def ensure_migrating_with_insufficient_user_permissions_fails(
        source_bs, dest_bs, upload_tools, tmp_dir):
    """Ensure migration fails when a user does not have the right permissions.

    A non-superuser on a controller cannot migrate their models between
    controllers.

    """
    source_client, dest_client = create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, 'failuser', 'addmodel')

    charm_path = local_charm_path(
        charm='dummy-source', juju_ver=source_client.version)
    source_client.deploy(charm_path)
    source_client.wait_for_started()

    log.info('Attempting migration process')

    expect_migration_attempt_to_fail(
        source_client,
        dest_client)

def ensure_migrating_with_superuser_user_permissions_succeeds(
        source_bs, dest_bs, upload_tools, tmp_dir):
    """Ensure migration succeeds when a user has superuser permissions

    A user with superuser permissions is able to migrate between controllers.

    """
    source_client, dest_client = create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, 'passuser', 'superuser')

    charm_path = local_charm_path(
        charm='dummy-source', juju_ver=source_client.version)
    source_client.deploy(charm_path)
    source_client.wait_for_started()

    log.info('Attempting migration process')

    migrate_model_to_controller(source_client, dest_client)

def create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, username, permission):
    # Create a user for both controllers that only has addmodel
    # permissions not superuser.
    new_user_home = os.path.join(tmp_dir, username)
    os.makedirs(new_user_home)
    new_user = User(username, 'write', [])
    normal_user_client_1 = source_bs.client.register_user(
        new_user, new_user_home)
    source_bs.client.grant(new_user.name, permission)

    second_controller_name = '{}_controllerb'.format(new_user.name)
    dest_client = dest_bs.client.register_user(
        new_user,
        new_user_home,
        controller_name=second_controller_name)
    dest_bs.client.grant(new_user.name, permission)

    source_client = normal_user_client_1.add_model(
        normal_user_client_1.env.clone('model-a'))

    return source_client, dest_client

def expect_migration_attempt_to_fail(source_client, dest_client):
    """Ensure that the migration attempt fails due to permissions.

    As we're capturing the stderr output it after we're done with it so it
    appears in test logs.

    """
    try:
        args = ['-c', source_client.env.controller.name,
                source_client.env.environment,
                dest_client.env.controller.name]
        log_output = source_client.get_juju_output(
            'migrate', *args, merge_stderr=True, include_e=False)
    except CalledProcessError as e:
        print(e.output, file=sys.stderr)
        if 'permission denied' not in e.output:
            raise
        log.info('SUCCESS: Migrate command failed as expected.')
    else:
        print(log_output, file=sys.stderr)
        raise JujuAssertionError('Migration did not fail as expected.')

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    bs1, bs2 = get_bootstrap_managers(args)

    assess_model_migration(bs1, bs2, args.upload_tools)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<commit_msg>Use of remote<commit_after>#!/usr/bin/env python
"""Tests for the Model Migration feature"""

from __future__ import print_function

import argparse
from contextlib import contextmanager
import logging
import os
from subprocess import CalledProcessError
import sys
from time import sleep

from assess_user_grant_revoke import User
from deploy_stack import BootstrapManager
from jujucharm import local_charm_path
from remote import remote_from_address
from utility import (
    JujuAssertionError,
    add_basic_testing_arguments,
    configure_logging,
    temp_dir,
    until_timeout,
)

__metaclass__ = type

log = logging.getLogger("assess_model_migration")

def assess_model_migration(bs1, bs2, upload_tools):
    with bs1.booted_context(upload_tools):
        bs1.client.enable_feature('migration')

        bs2.client.env.juju_home = bs1.client.env.juju_home
        with bs2.existing_booted_context(upload_tools):
            ensure_able_to_migrate_model_between_controllers(
                bs1, bs2, upload_tools)

            with temp_dir() as temp:
                ensure_migrating_with_insufficient_user_permissions_fails(
                    bs1, bs2, upload_tools, temp)
                ensure_migrating_with_superuser_user_permissions_succeeds(
                    bs1, bs2, upload_tools, temp)

            ensure_migration_rolls_back_on_failure(bs1, bs2, upload_tools)

def parse_args(argv):
    """Parse all arguments."""
    parser = argparse.ArgumentParser(
        description="Test model migration feature"
    )
    add_basic_testing_arguments(parser)
    return parser.parse_args(argv)

def get_bootstrap_managers(args):
    """Create 2 bootstrap managers from the provided args.

    Need to make a couple of elements uniqe (e.g. environment name) so we can
    have 2 bootstrapped at the same time.

    """
    bs_1 = BootstrapManager.from_args(args)
    bs_2 = BootstrapManager.from_args(args)

    # Give the second a separate/unique name.
    bs_2.temp_env_name = '{}-b'.format(bs_1.temp_env_name)

    bs_1.log_dir = _new_log_dir(args.logs, 'a')
    bs_2.log_dir = _new_log_dir(args.logs, 'b')

    return bs_1, bs_2

def _new_log_dir(log_dir, post_fix):
    new_log_dir = os.path.join(log_dir, 'env-{}'.format(post_fix))
    os.mkdir(new_log_dir)
    return new_log_dir

def wait_for_model(client, model_name, timeout=60):
    """Wait for a given `timeout` for a model of `model_name` to appear within
    `client`.

    Defaults to 10 seconds timeout.
    :raises AssertionError: If the named model does not appear in the specified
      timeout.

    """
    with client.check_timeouts():
        with client.ignore_soft_deadline():
            for _ in until_timeout(timeout):
                models = client.get_models()
                if model_name in [m['name'] for m in models['models']]:
                    return
                sleep(1)
            raise JujuAssertionError(
                'Model \'{}\' failed to appear after {} seconds'.format(
                    model_name, timeout
                ))

def test_deployed_mongo_is_up(client):
    """Ensure the mongo service is running as expected."""
    try:
        output = client.get_juju_output(
            'run', '--unit', 'mongodb/0', 'mongo --eval "db.getMongo()"')
        if 'connecting to: test' in output:
            return
    except CalledProcessError as e:
        # Pass through to assertion error
        log.error('Mongodb check command failed: {}'.format(e))
    raise AssertionError('Mongo db is not in an expected state.')

def ensure_able_to_migrate_model_between_controllers(
        source_environ, dest_environ, upload_tools):
    """Test simple migration of a model to another controller.

    Ensure that migration a model that has an application deployed upon it is
    able to continue it's operation after the migration process.

    Given 2 bootstrapped environments:
      - Deploy an application
        - ensure it's operating as expected
      - Migrate that model to the other environment
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional
      - Migrate the model back to the original environment
        - Note: Test for lp:1607457
        - Ensure it's operating as expected
        - Add a new unit to the application to ensure the model is functional

    """
    bundle = 'mongodb'
    application = 'mongodb'

    log.info('Deploying charm')
    # Don't move the default model so we can reuse it in later tests.
    test_model = source_environ.client.add_model(
        source_environ.client.env.clone('example-model'))
    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    log.info('Initiating migration process')

    migration_target_client = migrate_model_to_controller(
        test_model, dest_environ.client)

    migration_target_client.wait_for_workloads()
    test_deployed_mongo_is_up(migration_target_client)
    ensure_model_is_functional(migration_target_client, application)

    migration_target_client.remove_service(application)

def migrate_model_to_controller(source_client, dest_client):
    source_client.controller_juju(
        'migrate',
        (source_client.env.environment,
         dest_client.env.controller.name))

    migration_target_client = dest_client.clone(
        dest_client.env.clone(
            source_client.env.environment))

    wait_for_model(
        migration_target_client, source_client.env.environment)

    migration_target_client.wait_for_started()

    return migration_target_client

def ensure_model_is_functional(client, application):
    """Ensures that the migrated model is functional

    Add unit to application to ensure the model is contactable and working.
    Ensure that added unit is created on a new machine (check for bug
    LP:1607599)

    """
    client.juju('add-unit', (application,))
    client.wait_for_started()

    assert_units_on_different_machines(client, application)

def assert_units_on_different_machines(client, application):
    status = client.get_status()
    unit_machines = [u[1]['machine'] for u in status.iter_units()]

    raise_if_shared_machines(unit_machines)

def raise_if_shared_machines(unit_machines):
    """Raise an exception if `unit_machines` contain double ups of machine ids.

    A unique list of machine ids will be equal in length to the set of those
    machine ids.

    :raises ValueError: if an empty list is passed in.
    :raises JujuAssertionError: if any double-ups of machine ids are detected.

    """
    if not unit_machines:
        raise ValueError('Cannot share 0 machines. Empty list provided.')
    if len(unit_machines) != len(set(unit_machines)):
        raise JujuAssertionError('Appliction units reside on the same machine')

def ensure_migration_rolls_back_on_failure(source_bs, dest_bs, upload_tools):
    """Must successfully roll back migration when migration fails.

    If the target controller becomes unavailable for the migration to complete
    the migration must roll back.

    """
    source_client = source_bs.client
    dest_client = dest_bs.client

    bundle = 'mongodb'
    application = 'mongodb'

    test_model = source_client.add_model(
        source_client.env.clone('example-model'))

    test_model.juju("deploy", (bundle))
    test_model.wait_for_started()
    test_model.wait_for_workloads()
    test_deployed_mongo_is_up(test_model)

    test_model.controller_juju(
        'migrate',
        (test_model.env.environment,
         dest_client.env.controller.name))
    # Immediately disrupt the migration.
    with disable_apiserver(dest_client.get_controller_client()):
        # Wait for model to be back and working on the original controller.
        wait_for_model(test_model, source_client.env.environment)
        test_model.wait_for_started()
        test_deployed_mongo_is_up(test_model)
        ensure_model_is_functional(test_model, application)

@contextmanager
def disable_apiserver(admin_client):
    status = admin_client.get_status()
    controller_ip = status.get_machine_dns_name('0')
    rem_client = remote_from_address(controller_ip)
    try:
        rem_client.run('sudo service jujud-machine-0 stop')
        yield
    finally:
        rem_client.run('sudo service jujud-machine-0 start')

def ensure_migrating_with_insufficient_user_permissions_fails(
        source_bs, dest_bs, upload_tools, tmp_dir):
    """Ensure migration fails when a user does not have the right permissions.

    A non-superuser on a controller cannot migrate their models between
    controllers.

    """
    source_client, dest_client = create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, 'failuser', 'addmodel')

    charm_path = local_charm_path(
        charm='dummy-source', juju_ver=source_client.version)
    source_client.deploy(charm_path)
    source_client.wait_for_started()

    log.info('Attempting migration process')

    expect_migration_attempt_to_fail(
        source_client,
        dest_client)

def ensure_migrating_with_superuser_user_permissions_succeeds(
        source_bs, dest_bs, upload_tools, tmp_dir):
    """Ensure migration succeeds when a user has superuser permissions

    A user with superuser permissions is able to migrate between controllers.

    """
    source_client, dest_client = create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, 'passuser', 'superuser')

    charm_path = local_charm_path(
        charm='dummy-source', juju_ver=source_client.version)
    source_client.deploy(charm_path)
    source_client.wait_for_started()

    log.info('Attempting migration process')

    migrate_model_to_controller(source_client, dest_client)

def create_user_on_controllers(
        source_bs, dest_bs, tmp_dir, username, permission):
    # Create a user for both controllers that only has addmodel
    # permissions not superuser.
    new_user_home = os.path.join(tmp_dir, username)
    os.makedirs(new_user_home)
    new_user = User(username, 'write', [])
    normal_user_client_1 = source_bs.client.register_user(
        new_user, new_user_home)
    source_bs.client.grant(new_user.name, permission)

    second_controller_name = '{}_controllerb'.format(new_user.name)
    dest_client = dest_bs.client.register_user(
        new_user,
        new_user_home,
        controller_name=second_controller_name)
    dest_bs.client.grant(new_user.name, permission)

    source_client = normal_user_client_1.add_model(
        normal_user_client_1.env.clone('model-a'))

    return source_client, dest_client

def expect_migration_attempt_to_fail(source_client, dest_client):
    """Ensure that the migration attempt fails due to permissions.

    As we're capturing the stderr output it after we're done with it so it
    appears in test logs.

    """
    try:
        args = ['-c', source_client.env.controller.name,
                source_client.env.environment,
                dest_client.env.controller.name]
        log_output = source_client.get_juju_output(
            'migrate', *args, merge_stderr=True, include_e=False)
    except CalledProcessError as e:
        print(e.output, file=sys.stderr)
        if 'permission denied' not in e.output:
            raise
        log.info('SUCCESS: Migrate command failed as expected.')
    else:
        print(log_output, file=sys.stderr)
        raise JujuAssertionError('Migration did not fail as expected.')

def main(argv=None):
    args = parse_args(argv)
    configure_logging(args.verbose)

    bs1, bs2 = get_bootstrap_managers(args)

    assess_model_migration(bs1, bs2, args.upload_tools)

    return 0

if __name__ == '__main__':
    sys.exit(main())
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
# generated by wxGlade 0.6.3 on Wed Oct 15 18:55:59 2008

import wx
import csnBuild

# begin wxGlade: extracode
# end wxGlade

class Dialog(wx.Dialog):
    def __init__(self, *args, **kwds):
        # begin wxGlade: Dialog.__init__
        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        wx.Dialog.__init__(self, *args, **kwds)
        self.checkbox_1 = wx.CheckBox(self, -1, "checkbox_1")
        self.btnClose = wx.Button(self, -1, "Close")

        self.__set_properties()
        self.__do_layout()

        self.Bind(wx.EVT_BUTTON, self.OnClose, self.btnClose)
        # end wxGlade

        self.Bind(wx.EVT_CLOSE, self.OnClose, self)

    def __set_properties(self):
        # begin wxGlade: Dialog.__set_properties
        self.SetTitle("Select project types to be included")
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: Dialog.__do_layout
        sizerItems = wx.BoxSizer(wx.VERTICAL)
        sizerItems.Add(self.checkbox_1, 0, 0, 0)
        sizerItems.Add(self.btnClose, 1, wx.ALL|wx.EXPAND, 5)
        self.SetSizer(sizerItems)
        sizerItems.Fit(self)
        self.Layout()
        # end wxGlade
        self.sizerItems = sizerItems

    def ShowItems(self, items, filter):
        self.sizerItems.Clear()
        self.checkbox_1.Hide()
        self.checkBoxes = dict()
        self.filter = filter
        for item in sorted(items):
            self.checkBoxes[item] = wx.CheckBox(self, -1, item)
            self.checkBoxes[item].SetValue( not item in filter )
            self.sizerItems.Add(self.checkBoxes[item], 0, 0, 0)

        for super in csnBuild.globalSettings.subCategoriesOf.keys():
            value = True
            for sub in csnBuild.globalSettings.subCategoriesOf[super]:
                    value = value and (not sub in filter)

            self.checkBoxes[super] = wx.CheckBox(self, -1, super)
            self.checkBoxes[super].SetValue( value )
            self.sizerItems.Insert(0, self.checkBoxes[super], 0, 0, 0)
            self.Bind(wx.EVT_CHECKBOX, self.OnSuperCategoryCheckBoxChanged, self.checkBoxes[super])

        self.sizerItems.Add(self.btnClose, 1, wx.ALL|wx.EXPAND, 5)
        self.sizerItems.Fit(self)
        self.Layout()
        return self.ShowModal()

    def OnSuperCategoryCheckBoxChanged(self, event): # wxGlade: CSnakeOptionsFrame.<event_handler>
        for super in csnBuild.globalSettings.subCategoriesOf.keys():
            value = self.checkBoxes[super].GetValue()

            for cbName in self.checkBoxes.keys():
                if cbName in csnBuild.globalSettings.subCategoriesOf[super]:
                    self.checkBoxes[cbName].SetValue(value)

    def OnClose(self, event): # wxGlade: Dialog.<event_handler>
        while len(self.filter):
            self.filter.pop()
        for item in self.checkBoxes.keys():
            if not self.checkBoxes[item].GetValue():
                self.filter.append(item)
        self.MakeModal(0)
        self.Destroy()

# end of class Dialog

if __name__ == "__main__":
    app = wx.PySimpleApp(0)
    wx.InitAllImageHandlers()
    dialog = Dialog(None, -1, "")
    app.SetTopWindow(dialog)
    dialog.Show()
    app.MainLoop()
<commit_msg>Fix: previously filtered projects are now remembered correctly<commit_after>#!/usr/bin/env python
# -*- coding: iso-8859-1 -*-
# generated by wxGlade 0.6.3 on Wed Oct 15 18:55:59 2008

import wx
import csnBuild

# begin wxGlade: extracode
# end wxGlade

class Dialog(wx.Dialog):
    def __init__(self, *args, **kwds):
        # begin wxGlade: Dialog.__init__
        kwds["style"] = wx.DEFAULT_DIALOG_STYLE
        wx.Dialog.__init__(self, *args, **kwds)
        self.checkbox_1 = wx.CheckBox(self, -1, "checkbox_1")
        self.btnClose = wx.Button(self, -1, "Close")

        self.__set_properties()
        self.__do_layout()

        self.Bind(wx.EVT_BUTTON, self.OnClose, self.btnClose)
        # end wxGlade

        self.Bind(wx.EVT_CLOSE, self.OnClose, self)

    def __set_properties(self):
        # begin wxGlade: Dialog.__set_properties
        self.SetTitle("Select project types to be included")
        # end wxGlade

    def __do_layout(self):
        # begin wxGlade: Dialog.__do_layout
        sizerItems = wx.BoxSizer(wx.VERTICAL)
        sizerItems.Add(self.checkbox_1, 0, 0, 0)
        sizerItems.Add(self.btnClose, 1, wx.ALL|wx.EXPAND, 5)
        self.SetSizer(sizerItems)
        sizerItems.Fit(self)
        self.Layout()
        # end wxGlade
        self.sizerItems = sizerItems

    def ShowItems(self, categories, filter):
        self.sizerItems.Clear()
        self.checkbox_1.Hide()
        self.checkBoxes = dict()
        self.filter = filter
        for category in sorted(categories):
            self.checkBoxes[category] = wx.CheckBox(self, -1, category)
            self.checkBoxes[category].SetValue( not category in filter )
            self.sizerItems.Add(self.checkBoxes[category], 0, 0, 0)

        for super in csnBuild.globalSettings.subCategoriesOf.keys():
            value = True
            for sub in csnBuild.globalSettings.subCategoriesOf[super]:
                    value = value and (not sub in filter)

            self.checkBoxes[super] = wx.CheckBox(self, -1, super)
            self.checkBoxes[super].SetValue( value )
            self.sizerItems.Insert(0, self.checkBoxes[super], 0, 0, 0)
            self.Bind(wx.EVT_CHECKBOX, self.OnSuperCategoryCheckBoxChanged, self.checkBoxes[super])

        self.sizerItems.Add(self.btnClose, 1, wx.ALL|wx.EXPAND, 5)
        self.sizerItems.Fit(self)
        self.Layout()
        return self.ShowModal()

    def OnSuperCategoryCheckBoxChanged(self, event): # wxGlade: CSnakeOptionsFrame.<event_handler>
        for super in csnBuild.globalSettings.subCategoriesOf.keys():
            value = self.checkBoxes[super].GetValue()

            for cbName in self.checkBoxes.keys():
                if cbName in csnBuild.globalSettings.subCategoriesOf[super]:
                    self.checkBoxes[cbName].SetValue(value)

    def OnClose(self, event): # wxGlade: Dialog.<event_handler>
        for category in self.checkBoxes.keys():
            if category in self.filter:
                self.filter.remove(category)
            if not self.checkBoxes[category].GetValue():
                self.filter.append(category)
        self.MakeModal(0)
        self.Destroy()

# end of class Dialog

if __name__ == "__main__":
    app = wx.PySimpleApp(0)
    wx.InitAllImageHandlers()
    dialog = Dialog(None, -1, "")
    app.SetTopWindow(dialog)
    dialog.Show()
    app.MainLoop()
<|endoftext|>


<commit_before># pylint: disable=W0401

import searchpath
import pathutils
import pauser
import utils
import easylsq
import basicfit
import rc
import errorvalue
import arithmetic

from searchpath import *
from pathutils import *
from pauser import *
from utils import *
from easylsq import *
from basicfit import *
from errorvalue import *
from arithmetic import *
from rc import sastoolrc
HC = 12398.419 #Planck's constant times speed of light, in eV*Angstrom units

class SASException(Exception):
    "General exception class for the package `sastool`"
    pass

__all__ = ['arithmetic', 'basicfit', 'easylsq', 'errorvalue', 'pathutils',
           'pauser', 'rc', 'searchpath', 'utils']

for k in __all__[:]:
    __all__.extend(eval('%s.__all__' % k))
<commit_msg>Re-based SASException to BaseException<commit_after># pylint: disable=W0401

import searchpath
import pathutils
import pauser
import utils
import easylsq
import basicfit
import rc
import errorvalue
import arithmetic

from searchpath import *
from pathutils import *
from pauser import *
from utils import *
from easylsq import *
from basicfit import *
from errorvalue import *
from arithmetic import *
from rc import sastoolrc
HC = 12398.419 #Planck's constant times speed of light, in eV*Angstrom units

class SASException(BaseException):
    "General exception class for the package `sastool`"
    pass

__all__ = ['arithmetic', 'basicfit', 'easylsq', 'errorvalue', 'pathutils',
           'pauser', 'rc', 'searchpath', 'utils']

for k in __all__[:]:
    __all__.extend(eval('%s.__all__' % k))
<|endoftext|>