<commit_before># -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Winstackers Release Notes documentation build configuration file, created by
# sphinx-quickstart on Tue Nov  3 17:40:50 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'openstackdocstheme',
    'reno.sphinxext',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = u'OS-WIN Release Notes'
copyright = u'2015, Winstackers Developers'

# openstackdocstheme options
openstackdocs_repo_name = 'openstack/os-win'
openstackdocs_auto_name = False
openstackdocs_bug_project = 'os-win'
openstackdocs_bug_tag = ''

# Release notes do not need a version in the title, they span
# multiple versions.
# The full version, including alpha/beta/rc tags.
release = ''
# The short X.Y version.
version = ''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'native'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'openstackdocs'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'OSWINReleaseNotesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'OSWINReleaseNotes.tex', u'OS-WIN Release Notes Documentation',
     u'Winstackers Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'oswinreleasenotes', u'OS-WIN Release Notes Documentation',
     [u'Winstackers Developers'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'OSWINReleaseNotes', u'OS-WIN Release Notes Documentation',
     u'Winstackers Developers', 'OSWINReleaseNotes',
     'Windows / Hyper-V management lib.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# -- Options for Internationalization output ------------------------------
locale_dirs = ['locale/']
<commit_msg>remove unicode prefix from code<commit_after># -*- coding: utf-8 -*-
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Winstackers Release Notes documentation build configuration file, created by
# sphinx-quickstart on Tue Nov  3 17:40:50 2015.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
# sys.path.insert(0, os.path.abspath('.'))

# -- General configuration ------------------------------------------------

# If your documentation needs a minimal Sphinx version, state it here.
# needs_sphinx = '1.0'

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'openstackdocstheme',
    'reno.sphinxext',
]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# The suffix of source filenames.
source_suffix = '.rst'

# The encoding of source files.
# source_encoding = 'utf-8-sig'

# The master toctree document.
master_doc = 'index'

# General information about the project.
project = 'OS-WIN Release Notes'
copyright = '2015, Winstackers Developers'

# openstackdocstheme options
openstackdocs_repo_name = 'openstack/os-win'
openstackdocs_auto_name = False
openstackdocs_bug_project = 'os-win'
openstackdocs_bug_tag = ''

# Release notes do not need a version in the title, they span
# multiple versions.
# The full version, including alpha/beta/rc tags.
release = ''
# The short X.Y version.
version = ''

# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
# language = None

# There are two options for replacing |today|: either, you set today to some
# non-false value, then it is used:
# today = ''
# Else, today_fmt is used as the format for a strftime call.
# today_fmt = '%B %d, %Y'

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
exclude_patterns = []

# The reST default role (used for this markup: `text`) to use for all
# documents.
# default_role = None

# If true, '()' will be appended to :func: etc. cross-reference text.
# add_function_parentheses = True

# If true, the current module name will be prepended to all description
# unit titles (such as .. function::).
# add_module_names = True

# If true, sectionauthor and moduleauthor directives will be shown in the
# output. They are ignored by default.
# show_authors = False

# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'native'

# A list of ignored prefixes for module index sorting.
# modindex_common_prefix = []

# If true, keep warnings as "system message" paragraphs in the built documents.
# keep_warnings = False

# -- Options for HTML output ----------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
html_theme = 'openstackdocs'

# Theme options are theme-specific and customize the look and feel of a theme
# further.  For a list of options available for each theme, see the
# documentation.
# html_theme_options = {}

# Add any paths that contain custom themes here, relative to this directory.
# html_theme_path = []

# The name for this set of Sphinx documents.  If None, it defaults to
# "<project> v<release> documentation".
# html_title = None

# A shorter title for the navigation bar.  Default is the same as html_title.
# html_short_title = None

# The name of an image file (relative to this directory) to place at the top
# of the sidebar.
# html_logo = None

# The name of an image file (within the static path) to use as favicon of the
# docs.  This file should be a Windows icon file (.ico) being 16x16 or 32x32
# pixels large.
# html_favicon = None

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

# Add any extra paths that contain custom files (such as robots.txt or
# .htaccess) here, relative to this directory. These files are copied
# directly to the root of the documentation.
# html_extra_path = []

# If not '', a 'Last updated on:' timestamp is inserted at every page bottom,
# using the given strftime format.
# html_last_updated_fmt = '%b %d, %Y'

# If true, SmartyPants will be used to convert quotes and dashes to
# typographically correct entities.
# html_use_smartypants = True

# Custom sidebar templates, maps document names to template names.
# html_sidebars = {}

# Additional templates that should be rendered to pages, maps page names to
# template names.
# html_additional_pages = {}

# If false, no module index is generated.
# html_domain_indices = True

# If false, no index is generated.
# html_use_index = True

# If true, the index is split into individual pages for each letter.
# html_split_index = False

# If true, links to the reST sources are added to the pages.
# html_show_sourcelink = True

# If true, "Created using Sphinx" is shown in the HTML footer. Default is True.
# html_show_sphinx = True

# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True.
# html_show_copyright = True

# If true, an OpenSearch description file will be output, and all pages will
# contain a <link> tag referring to it.  The value of this option must be the
# base URL from which the finished HTML is served.
# html_use_opensearch = ''

# This is the file name suffix for HTML files (e.g. ".xhtml").
# html_file_suffix = None

# Output file base name for HTML help builder.
htmlhelp_basename = 'OSWINReleaseNotesdoc'

# -- Options for LaTeX output ---------------------------------------------

latex_elements = {
    # The paper size ('letterpaper' or 'a4paper').
    # 'papersize': 'letterpaper',

    # The font size ('10pt', '11pt' or '12pt').
    # 'pointsize': '10pt',

    # Additional stuff for the LaTeX preamble.
    # 'preamble': '',
}

# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
#  author, documentclass [howto, manual, or own class]).
latex_documents = [
    ('index', 'OSWINReleaseNotes.tex', 'OS-WIN Release Notes Documentation',
     'Winstackers Developers', 'manual'),
]

# The name of an image file (relative to this directory) to place at the top of
# the title page.
# latex_logo = None

# For "manual" documents, if this is true, then toplevel headings are parts,
# not chapters.
# latex_use_parts = False

# If true, show page references after internal links.
# latex_show_pagerefs = False

# If true, show URL addresses after external links.
# latex_show_urls = False

# Documents to append as an appendix to all manuals.
# latex_appendices = []

# If false, no module index is generated.
# latex_domain_indices = True

# -- Options for manual page output ---------------------------------------

# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
    ('index', 'oswinreleasenotes', 'OS-WIN Release Notes Documentation',
     ['Winstackers Developers'], 1)
]

# If true, show URL addresses after external links.
# man_show_urls = False

# -- Options for Texinfo output -------------------------------------------

# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
#  dir menu entry, description, category)
texinfo_documents = [
    ('index', 'OSWINReleaseNotes', 'OS-WIN Release Notes Documentation',
     'Winstackers Developers', 'OSWINReleaseNotes',
     'Windows / Hyper-V management lib.',
     'Miscellaneous'),
]

# Documents to append as an appendix to all manuals.
# texinfo_appendices = []

# If false, no module index is generated.
# texinfo_domain_indices = True

# How to display URL addresses: 'footnote', 'no', or 'inline'.
# texinfo_show_urls = 'footnote'

# If true, do not generate a @detailmenu in the "Top" node's menu.
# texinfo_no_detailmenu = False

# -- Options for Internationalization output ------------------------------
locale_dirs = ['locale/']
<|endoftext|>


<commit_before>from numpy.testing import NumpyTest

import PyMC2
# import tests

__all__=['test']
def test():
    for m in PyMC2.tests.__modules__:
        print 'Testing ' + m
        mod = 'tests.%s'%m
        print mod
        testsuite = NumpyTest(mod)
        testsuite.test()

if __name__=='__main__':
    test()

<commit_msg>fixed bug in testsuite<commit_after>from numpy.testing import NumpyTest

import PyMC2.tests
# import tests

__all__=['test']
def test():
    for m in PyMC2.tests.__modules__:
        print 'Testing ' + m
        mod = 'tests.%s'%m
        print mod
        testsuite = NumpyTest(mod)
        testsuite.test()

if __name__=='__main__':
    test()

<|endoftext|>


<commit_before>import json

import os
from copy import deepcopy

from git import Repo
from jinja2 import Environment, FileSystemLoader, StrictUndefined

from releasewarrior.click_input import generate_inflight_task_from_input, is_future_threat_input
from releasewarrior.click_input import generate_prereq_task_from_input
from releasewarrior.click_input import generate_inflight_issue_from_input
from releasewarrior.collections import Release
from releasewarrior.git import commit, push
from releasewarrior.helpers import get_branch, load_json, get_remaining_items

def order_data(data):
    # order prereqs by deadline
    prereqs = data["preflight"]["human_tasks"]
    data["preflight"]["human_tasks"] = sorted(prereqs, key=lambda x: x["deadline"])
    # order buildnums by most recent
    builds = data["inflight"]
    data["inflight"] = sorted(builds, key=lambda x: x["buildnum"], reverse=True)
    return data

def get_current_build_index(release):
    highest_buildnum = 0
    return_index = 0
    for index, build in enumerate(release['inflight']):
        if build["buildnum"] > highest_buildnum:
            highest_buildnum = build["buildnum"]
            return_index = index
    return return_index

def generate_wiki(data, wiki_template, logger, config):
    logger.info("generating wiki from template and config")

    if not wiki_template:
        wiki_template = config['templates']["wiki"]["generic"]

    env = Environment(loader=FileSystemLoader(config['templates_dir']),
                      undefined=StrictUndefined, trim_blocks=True)

    template = env.get_template(wiki_template)
    return template.render(**data)

def write_data(data_path, content, logger, config):
    logger.info("writing to data file: %s", data_path)
    with open(data_path, 'w') as data_file:
        json.dump(content, data_file, indent=4, sort_keys=True)

    return data_path

def write_corsica(corsica_path, content, logger, config):
    logger.info("writing to corsica file: %s", corsica_path)
    with open(corsica_path, 'w') as cp:
        cp.write(content)

    return corsica_path

def write_wiki(wiki_path, content, logger, config):
    logger.info("writing to wiki file: %s", wiki_path)
    with open(wiki_path, 'w') as wp:
        wp.write(content)

    return wiki_path

def get_release_files(release, logging, config):
    upcoming_path = os.path.join(config['releasewarrior_data_repo'],
                                 config['releases']['upcoming'][release.product])
    inflight_path = os.path.join(config['releasewarrior_data_repo'],
                                 config['releases']['inflight'][release.product])
    data_file = "{}-{}-{}.json".format(release.product, release.branch, release.version)
    wiki_file = "{}-{}-{}.md".format(release.product, release.branch, release.version)
    release_path = upcoming_path
    if os.path.exists(os.path.join(inflight_path, data_file)):
        release_path = inflight_path
    return [
        os.path.join(release_path, data_file),
        os.path.join(release_path, wiki_file)
    ]

def complete_filter(tasks, issues, aborted):
    return (all(task["resolved"] for task in tasks) or aborted) and \
            all(issue["resolved"] for issue in issues)

def incomplete_filter(tasks, issues, aborted):
    return (not all(task["resolved"] for task in tasks) and not aborted) or \
            not all(issue["resolved"] for issue in issues)

def no_filter(tasks, issues, aborted):
    return True

def get_releases(config, logger, inflight=True, filter=no_filter):
    for release_path in config['releases']['inflight' if inflight else 'upcoming'].values():
        search_dir = os.path.join(config['releasewarrior_data_repo'], release_path)
        for root, dirs, files in os.walk(search_dir):
            for f in [data_file for data_file in files if data_file.endswith(".json")]:
                abs_f = os.path.join(search_dir, f)
                with open(abs_f) as data_f:
                    data = json.load(data_f)
                    if inflight:
                        tasks = data["inflight"][get_current_build_index(data)]["human_tasks"]
                        issues = data["inflight"][get_current_build_index(data)]["issues"]
                        aborted = data["inflight"][get_current_build_index(data)]["aborted"]
                    else:
                        tasks = data["preflight"]["human_tasks"]
                        issues = []
                        aborted = False
                    if filter(tasks, issues, aborted):
                        yield data

def get_release_info(product, version, logger, config):
    branch = get_branch(version, product, logger)
    release = Release(product=product, version=version, branch=branch)
    data_path, wiki_path = get_release_files(release, logger, config)
    logger.debug("release info: %s", release)
    logger.debug("data path: %s", data_path)
    logger.debug("wiki path: %s", wiki_path)
    return release, data_path, wiki_path

def generate_corsica(config, logger):
    all_inflight_releases = get_releases(config, logger)
    corsica_data = {
        "releases": {}
    }
    for release in all_inflight_releases:
        branch = get_branch(release["version"], release["product"], logger)
        branch = branch.replace("-rc", "")
        human_tasks = {}
        current_build_index = get_current_build_index(release)
        for task in release["inflight"][current_build_index]["human_tasks"]:
            if task.get("alias"):
                human_tasks[task["alias"]] = task["resolved"]
        if not corsica_data["releases"].get(release["product"]):
            corsica_data["releases"][release["product"]] = {}
        corsica_data["releases"][release["product"]][branch] = {
            "buildnum": release["inflight"][current_build_index]["buildnum"],
            "version": release["version"].replace("rc", ""),
            "human_tasks": human_tasks
        }
    index_template = config['templates']["corsica"]["index"]

    env = Environment(loader=FileSystemLoader(config['templates_dir']),
                      undefined=StrictUndefined, trim_blocks=True)

    template = env.get_template(index_template)
    return template.render(**corsica_data)

def write_and_commit(data, data_path, wiki_path, commit_msg, logger, config, wiki_template=None):
    corsica_path = os.path.join(config["releasewarrior_data_repo"], config["corsica"])
    wiki = generate_wiki(data, wiki_template, logger, config)
    paths = []

    paths.append(write_data(data_path, data, logger, config))
    paths.append(write_wiki(wiki_path, wiki, logger, config))
    if config['corsica_enabled']:
        corsica = generate_corsica(config, logger)
        paths.append(write_corsica(corsica_path, corsica, logger, config))
    for p in paths:
        logger.debug(p)
    if not os.environ.get("RW_DEV"):
        commit(paths, commit_msg, logger, config)
        if config.get("auto_push_data"):
            push(logger, config)

def generate_newbuild_data(data, release, data_path, wiki_path, logger, config):
    is_first_gtb = "upcoming" in data_path
    current_build_index = get_current_build_index(data)
    if is_first_gtb:
        # resolve shipit task
        for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
            if task["alias"] == "shipit":
                data["inflight"][current_build_index]["human_tasks"][index]["resolved"] = True

        #   delete json and md files from upcoming dir, and set new dest paths to be inflight
        repo = Repo(config['releasewarrior_data_repo'])
        inflight_dir = os.path.join(config['releasewarrior_data_repo'],
                                    config['releases']['inflight'][release.product])
        moved_files = repo.index.move([data_path, wiki_path, inflight_dir])
        # set data and wiki paths to new dest (inflight) dir
        # moved_files is a list of tuples representing [files_moved][destination_location]
        # TODO
        data_path = os.path.join(config['releasewarrior_data_repo'], moved_files[0][1])
        wiki_path = os.path.join(config['releasewarrior_data_repo'], moved_files[1][1])
    else:
        #  kill latest buildnum add new buildnum based most recent buildnum
        logger.info("most recent buildnum has been aborted, starting a new buildnum")
        newbuild = deepcopy(data["inflight"][current_build_index])
        # abort the now previous buildnum
        data["inflight"][current_build_index]["aborted"] = True
        newbuild["aborted"] = False
        for task in newbuild["human_tasks"]:
            if task["alias"] == "shipit":
                continue  # leave submitted to shipit as resolved
            # reset all tasks to unresolved
            task["resolved"] = False
        # carry forward only unresolved issues
        newbuild["issues"] = [issue for issue in get_remaining_items(newbuild["issues"])]
        # increment buildnum
        newbuild["buildnum"] = newbuild["buildnum"] + 1
        # add new buildnum based on previous to current release
        data["inflight"].append(newbuild)
    current_build_index = get_current_build_index(data)

    return data, data_path, wiki_path

def get_tracking_release_data(release, gtb_date, logger, config):
    logger.info("generating data from template and config")
    data_template = os.path.join(
        config['templates_dir'],
        config['templates']["data"][release.product][release.branch]
    )
    data = load_json(data_template)
    data["version"] = release.version
    data["date"] = gtb_date
    return data

def update_inflight_human_tasks(data, resolve, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    if resolve:
        for human_task_id in resolve:
            # attempt to use id as alias
            for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
                if human_task_id == task['alias']:
                    data["inflight"][current_build_index]["human_tasks"][index]["resolved"] = True
                    break
            else:
                # use id as index
                # 0 based index so -1
                human_task_id = int(human_task_id) - 1
                data["inflight"][current_build_index]["human_tasks"][human_task_id]["resolved"] = True
    else:
        logger.info("Current existing inflight tasks:")
        for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
            logger.info("ID: %s - %s", index + 1, task["description"])
        # create a new inflight human task through interactive inputs
        new_human_task = generate_inflight_task_from_input()
        data["inflight"][current_build_index]["human_tasks"].insert(
            new_human_task.position,
            {
                "alias": "", "description": new_human_task.description,
                "docs": new_human_task.docs, "resolved": False
            }
        )

    return data

def update_prereq_human_tasks(data, resolve):
    data = deepcopy(data)
    if resolve:
        for human_task_id in resolve:
            # 0 based index so -1
            human_task_id = int(human_task_id) - 1
            data["preflight"]["human_tasks"][human_task_id]["resolved"] = True
    else:
        # create a new prerequisite task through interactive inputs
        new_prereq = generate_prereq_task_from_input(gtb_date=data.get('date'))
        data["preflight"]["human_tasks"].append(
            {
                "bug": new_prereq.bug, "deadline": new_prereq.deadline,
                "description": new_prereq.description, "resolved": False
            }
        )
    return data

def update_inflight_issue(data, resolve, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    if resolve:
        for issue_id in resolve:
            # 0 based index so -1
            issue_index = int(issue_id) - 1
            logger.info("resolving issue %s - %s", issue_id,
                        data["inflight"][current_build_index]["issues"][issue_index]["description"])
            data["inflight"][current_build_index]["issues"][issue_index]["resolved"] = True
            data["inflight"][current_build_index]["issues"][issue_index]["future_threat"] = is_future_threat_input()

    else:
        # create a new issueuisite task through interactive inputs
        new_issue = generate_inflight_issue_from_input()
        data["inflight"][current_build_index]["issues"].append(
            {
                "who": new_issue.who, "bug": new_issue.bug, "description": new_issue.description,
                "resolved": False, "future_threat": True
            }
        )
    return data

def update_inflight_graphid(data, phase, graphid, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    graphids = data["inflight"][current_build_index]["graphids"]
    existing_phases = [p for p, _ in graphids]
    if phase in existing_phases:
        i = existing_phases.index(phase)
        graphids[i] = [phase, graphid]
    else:
        graphids.append([phase, graphid])
    return data

def generate_release_postmortem_data(release):
    postmortem_release = {
        "version": release['version'],
        "product": release['product'],
        "date": release['date'],
    }
    future_threat_issues = []
    resolved_issues = []
    for build in release["inflight"]:
        for issue in build["issues"]:
            issue["buildnum"] = build["buildnum"]
            if issue["future_threat"]:
                future_threat_issues.append(issue)
            else:
                resolved_issues.append(issue)

    postmortem_release["future_threats"] = future_threat_issues
    postmortem_release["resolved"] = resolved_issues

    return postmortem_release

def log_release_status(release, logger):
    current_build_index = get_current_build_index(release)
    remaining_tasks = get_remaining_items(release["inflight"][current_build_index]["human_tasks"])
    remaining_issues = get_remaining_items(release["inflight"][current_build_index]["issues"])

    logger.info("=" * 79)
    logger.info("RELEASE: %s %s build%s %s",
                release["product"], release["version"],
                release["inflight"][current_build_index]["buildnum"], release["date"])
    for graph_info in release["inflight"][current_build_index]["graphids"]:
        logger.info("%s graph: https://tools.taskcluster.net/task-group-inspector/#/%s",
                    graph_info[0], graph_info[1])
    logger.info("\tIncomplete human tasks:")
    for task in remaining_tasks:
        alias = ""
        if task.get("alias"):
            alias = "(alias: {})".format(task["alias"])
        logger.info("\t\t* ID %s %s - %s", task["id"], alias, task["description"])
    logger.info("\tUnresolved issues:")
    for issue in remaining_issues:
        logger.info("\t\t* ID: %s bug: %s - %s", issue["id"], issue["bug"], issue["description"])
<commit_msg>Fix graphids being carried forward<commit_after>import json

import os
from copy import deepcopy

from git import Repo
from jinja2 import Environment, FileSystemLoader, StrictUndefined

from releasewarrior.click_input import generate_inflight_task_from_input, is_future_threat_input
from releasewarrior.click_input import generate_prereq_task_from_input
from releasewarrior.click_input import generate_inflight_issue_from_input
from releasewarrior.collections import Release
from releasewarrior.git import commit, push
from releasewarrior.helpers import get_branch, load_json, get_remaining_items

def order_data(data):
    # order prereqs by deadline
    prereqs = data["preflight"]["human_tasks"]
    data["preflight"]["human_tasks"] = sorted(prereqs, key=lambda x: x["deadline"])
    # order buildnums by most recent
    builds = data["inflight"]
    data["inflight"] = sorted(builds, key=lambda x: x["buildnum"], reverse=True)
    return data

def get_current_build_index(release):
    highest_buildnum = 0
    return_index = 0
    for index, build in enumerate(release['inflight']):
        if build["buildnum"] > highest_buildnum:
            highest_buildnum = build["buildnum"]
            return_index = index
    return return_index

def generate_wiki(data, wiki_template, logger, config):
    logger.info("generating wiki from template and config")

    if not wiki_template:
        wiki_template = config['templates']["wiki"]["generic"]

    env = Environment(loader=FileSystemLoader(config['templates_dir']),
                      undefined=StrictUndefined, trim_blocks=True)

    template = env.get_template(wiki_template)
    return template.render(**data)

def write_data(data_path, content, logger, config):
    logger.info("writing to data file: %s", data_path)
    with open(data_path, 'w') as data_file:
        json.dump(content, data_file, indent=4, sort_keys=True)

    return data_path

def write_corsica(corsica_path, content, logger, config):
    logger.info("writing to corsica file: %s", corsica_path)
    with open(corsica_path, 'w') as cp:
        cp.write(content)

    return corsica_path

def write_wiki(wiki_path, content, logger, config):
    logger.info("writing to wiki file: %s", wiki_path)
    with open(wiki_path, 'w') as wp:
        wp.write(content)

    return wiki_path

def get_release_files(release, logging, config):
    upcoming_path = os.path.join(config['releasewarrior_data_repo'],
                                 config['releases']['upcoming'][release.product])
    inflight_path = os.path.join(config['releasewarrior_data_repo'],
                                 config['releases']['inflight'][release.product])
    data_file = "{}-{}-{}.json".format(release.product, release.branch, release.version)
    wiki_file = "{}-{}-{}.md".format(release.product, release.branch, release.version)
    release_path = upcoming_path
    if os.path.exists(os.path.join(inflight_path, data_file)):
        release_path = inflight_path
    return [
        os.path.join(release_path, data_file),
        os.path.join(release_path, wiki_file)
    ]

def complete_filter(tasks, issues, aborted):
    return (all(task["resolved"] for task in tasks) or aborted) and \
            all(issue["resolved"] for issue in issues)

def incomplete_filter(tasks, issues, aborted):
    return (not all(task["resolved"] for task in tasks) and not aborted) or \
            not all(issue["resolved"] for issue in issues)

def no_filter(tasks, issues, aborted):
    return True

def get_releases(config, logger, inflight=True, filter=no_filter):
    for release_path in config['releases']['inflight' if inflight else 'upcoming'].values():
        search_dir = os.path.join(config['releasewarrior_data_repo'], release_path)
        for root, dirs, files in os.walk(search_dir):
            for f in [data_file for data_file in files if data_file.endswith(".json")]:
                abs_f = os.path.join(search_dir, f)
                with open(abs_f) as data_f:
                    data = json.load(data_f)
                    if inflight:
                        tasks = data["inflight"][get_current_build_index(data)]["human_tasks"]
                        issues = data["inflight"][get_current_build_index(data)]["issues"]
                        aborted = data["inflight"][get_current_build_index(data)]["aborted"]
                    else:
                        tasks = data["preflight"]["human_tasks"]
                        issues = []
                        aborted = False
                    if filter(tasks, issues, aborted):
                        yield data

def get_release_info(product, version, logger, config):
    branch = get_branch(version, product, logger)
    release = Release(product=product, version=version, branch=branch)
    data_path, wiki_path = get_release_files(release, logger, config)
    logger.debug("release info: %s", release)
    logger.debug("data path: %s", data_path)
    logger.debug("wiki path: %s", wiki_path)
    return release, data_path, wiki_path

def generate_corsica(config, logger):
    all_inflight_releases = get_releases(config, logger)
    corsica_data = {
        "releases": {}
    }
    for release in all_inflight_releases:
        branch = get_branch(release["version"], release["product"], logger)
        branch = branch.replace("-rc", "")
        human_tasks = {}
        current_build_index = get_current_build_index(release)
        for task in release["inflight"][current_build_index]["human_tasks"]:
            if task.get("alias"):
                human_tasks[task["alias"]] = task["resolved"]
        if not corsica_data["releases"].get(release["product"]):
            corsica_data["releases"][release["product"]] = {}
        corsica_data["releases"][release["product"]][branch] = {
            "buildnum": release["inflight"][current_build_index]["buildnum"],
            "version": release["version"].replace("rc", ""),
            "human_tasks": human_tasks
        }
    index_template = config['templates']["corsica"]["index"]

    env = Environment(loader=FileSystemLoader(config['templates_dir']),
                      undefined=StrictUndefined, trim_blocks=True)

    template = env.get_template(index_template)
    return template.render(**corsica_data)

def write_and_commit(data, data_path, wiki_path, commit_msg, logger, config, wiki_template=None):
    corsica_path = os.path.join(config["releasewarrior_data_repo"], config["corsica"])
    wiki = generate_wiki(data, wiki_template, logger, config)
    paths = []

    paths.append(write_data(data_path, data, logger, config))
    paths.append(write_wiki(wiki_path, wiki, logger, config))
    if config['corsica_enabled']:
        corsica = generate_corsica(config, logger)
        paths.append(write_corsica(corsica_path, corsica, logger, config))
    for p in paths:
        logger.debug(p)
    if not os.environ.get("RW_DEV"):
        commit(paths, commit_msg, logger, config)
        if config.get("auto_push_data"):
            push(logger, config)

def generate_newbuild_data(data, release, data_path, wiki_path, logger, config):
    is_first_gtb = "upcoming" in data_path
    current_build_index = get_current_build_index(data)
    if is_first_gtb:
        # resolve shipit task
        for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
            if task["alias"] == "shipit":
                data["inflight"][current_build_index]["human_tasks"][index]["resolved"] = True

        #   delete json and md files from upcoming dir, and set new dest paths to be inflight
        repo = Repo(config['releasewarrior_data_repo'])
        inflight_dir = os.path.join(config['releasewarrior_data_repo'],
                                    config['releases']['inflight'][release.product])
        moved_files = repo.index.move([data_path, wiki_path, inflight_dir])
        # set data and wiki paths to new dest (inflight) dir
        # moved_files is a list of tuples representing [files_moved][destination_location]
        # TODO
        data_path = os.path.join(config['releasewarrior_data_repo'], moved_files[0][1])
        wiki_path = os.path.join(config['releasewarrior_data_repo'], moved_files[1][1])
    else:
        #  kill latest buildnum add new buildnum based most recent buildnum
        logger.info("most recent buildnum has been aborted, starting a new buildnum")
        newbuild = deepcopy(data["inflight"][current_build_index])
        # abort the now previous buildnum
        data["inflight"][current_build_index]["aborted"] = True
        newbuild["aborted"] = False
        for task in newbuild["human_tasks"]:
            if task["alias"] == "shipit":
                continue  # leave submitted to shipit as resolved
            # reset all tasks to unresolved
            task["resolved"] = False
        # carry forward only unresolved issues
        newbuild["issues"] = [issue for issue in get_remaining_items(newbuild["issues"])]
        # increment buildnum
        newbuild["buildnum"] = newbuild["buildnum"] + 1
        # ignore old graphids
        newbuild["graphids"] = []
        # add new buildnum based on previous to current release
        data["inflight"].append(newbuild)
    current_build_index = get_current_build_index(data)

    return data, data_path, wiki_path

def get_tracking_release_data(release, gtb_date, logger, config):
    logger.info("generating data from template and config")
    data_template = os.path.join(
        config['templates_dir'],
        config['templates']["data"][release.product][release.branch]
    )
    data = load_json(data_template)
    data["version"] = release.version
    data["date"] = gtb_date
    return data

def update_inflight_human_tasks(data, resolve, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    if resolve:
        for human_task_id in resolve:
            # attempt to use id as alias
            for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
                if human_task_id == task['alias']:
                    data["inflight"][current_build_index]["human_tasks"][index]["resolved"] = True
                    break
            else:
                # use id as index
                # 0 based index so -1
                human_task_id = int(human_task_id) - 1
                data["inflight"][current_build_index]["human_tasks"][human_task_id]["resolved"] = True
    else:
        logger.info("Current existing inflight tasks:")
        for index, task in enumerate(data["inflight"][current_build_index]["human_tasks"]):
            logger.info("ID: %s - %s", index + 1, task["description"])
        # create a new inflight human task through interactive inputs
        new_human_task = generate_inflight_task_from_input()
        data["inflight"][current_build_index]["human_tasks"].insert(
            new_human_task.position,
            {
                "alias": "", "description": new_human_task.description,
                "docs": new_human_task.docs, "resolved": False
            }
        )

    return data

def update_prereq_human_tasks(data, resolve):
    data = deepcopy(data)
    if resolve:
        for human_task_id in resolve:
            # 0 based index so -1
            human_task_id = int(human_task_id) - 1
            data["preflight"]["human_tasks"][human_task_id]["resolved"] = True
    else:
        # create a new prerequisite task through interactive inputs
        new_prereq = generate_prereq_task_from_input(gtb_date=data.get('date'))
        data["preflight"]["human_tasks"].append(
            {
                "bug": new_prereq.bug, "deadline": new_prereq.deadline,
                "description": new_prereq.description, "resolved": False
            }
        )
    return data

def update_inflight_issue(data, resolve, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    if resolve:
        for issue_id in resolve:
            # 0 based index so -1
            issue_index = int(issue_id) - 1
            logger.info("resolving issue %s - %s", issue_id,
                        data["inflight"][current_build_index]["issues"][issue_index]["description"])
            data["inflight"][current_build_index]["issues"][issue_index]["resolved"] = True
            data["inflight"][current_build_index]["issues"][issue_index]["future_threat"] = is_future_threat_input()

    else:
        # create a new issueuisite task through interactive inputs
        new_issue = generate_inflight_issue_from_input()
        data["inflight"][current_build_index]["issues"].append(
            {
                "who": new_issue.who, "bug": new_issue.bug, "description": new_issue.description,
                "resolved": False, "future_threat": True
            }
        )
    return data

def update_inflight_graphid(data, phase, graphid, logger):
    data = deepcopy(data)
    current_build_index = get_current_build_index(data)
    graphids = data["inflight"][current_build_index]["graphids"]
    existing_phases = [p for p, _ in graphids]
    if phase in existing_phases:
        i = existing_phases.index(phase)
        graphids[i] = [phase, graphid]
    else:
        graphids.append([phase, graphid])
    return data

def generate_release_postmortem_data(release):
    postmortem_release = {
        "version": release['version'],
        "product": release['product'],
        "date": release['date'],
    }
    future_threat_issues = []
    resolved_issues = []
    for build in release["inflight"]:
        for issue in build["issues"]:
            issue["buildnum"] = build["buildnum"]
            if issue["future_threat"]:
                future_threat_issues.append(issue)
            else:
                resolved_issues.append(issue)

    postmortem_release["future_threats"] = future_threat_issues
    postmortem_release["resolved"] = resolved_issues

    return postmortem_release

def log_release_status(release, logger):
    current_build_index = get_current_build_index(release)
    remaining_tasks = get_remaining_items(release["inflight"][current_build_index]["human_tasks"])
    remaining_issues = get_remaining_items(release["inflight"][current_build_index]["issues"])

    logger.info("=" * 79)
    logger.info("RELEASE: %s %s build%s %s",
                release["product"], release["version"],
                release["inflight"][current_build_index]["buildnum"], release["date"])
    for graph_info in release["inflight"][current_build_index]["graphids"]:
        logger.info("%s graph: https://tools.taskcluster.net/task-group-inspector/#/%s",
                    graph_info[0], graph_info[1])
    logger.info("\tIncomplete human tasks:")
    for task in remaining_tasks:
        alias = ""
        if task.get("alias"):
            alias = "(alias: {})".format(task["alias"])
        logger.info("\t\t* ID %s %s - %s", task["id"], alias, task["description"])
    logger.info("\tUnresolved issues:")
    for issue in remaining_issues:
        logger.info("\t\t* ID: %s bug: %s - %s", issue["id"], issue["bug"], issue["description"])
<|endoftext|>


<commit_before>
# A consolidation of the data sources for the project
# includes:
# NetworkX!
# RDFlib!
# Other things!
#
# Works like Configure:
# Inherit from the Data class to access data of all kinds (listed above)

import sqlite3
import networkx as nx
import PyOpenWorm
from PyOpenWorm import Configureable, Configure, ConfigValue, BadConf
import hashlib
import csv
import urllib2
from rdflib import URIRef, Literal, Graph, Namespace, ConjunctiveGraph, BNode
from rdflib.namespace import RDFS,RDF
from datetime import datetime as DT
import datetime
from rdflib.namespace import XSD
from itertools import izip_longest
import os

# encapsulates some of the data all of the parts need...
class _B(ConfigValue):
    def __init__(self, f):
        self.v = False
        self.f = f

    def get(self):
        if not self.v:
            self.v = self.f()

        return self.v
    def invalidate(self):
        self.v = False

ZERO = datetime.timedelta(0)
class _UTC(datetime.tzinfo):
    """UTC"""

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO
utc = _UTC()

class _Z(ConfigValue):
    def __init__(self, c, n):
        self.n = n
    def get(self):
        return c[n]

propertyTypes = {"send" : 'http://openworm.org/entities/356',
        "Neuropeptide" : 'http://openworm.org/entities/354',
        "Receptor" : 'http://openworm.org/entities/361',
        "is a" : 'http://openworm.org/entities/1515',
        "neuromuscular junction" : 'http://openworm.org/entities/1516',
        "Innexin" : 'http://openworm.org/entities/355',
        "Neurotransmitter" : 'http://openworm.org/entities/313',
        "gap junction" : 'http://openworm.org/entities/357'}

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    while True:
        l = []
        try:
            for x in args:
                l.append(next(x))
        except:
            pass
        yield l
        if len(l) < n:
            break

class DataUser(Configureable):
    def __init__(self, conf = False):
        Configureable.__init__(self, conf=conf)
        if not isinstance(self.conf,Data):
            raise BadConf(self)

    @property
    def rdf(self):
        return self.conf['rdf.graph']

    def _remove_from_store(self, g):
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            s = " DELETE DATA {" + temp_graph.serialize(format="nt") + " } "
            self.conf['rdf.graph'].update(s)

    def _add_to_store(self, g, graph_name=False):
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            if graph_name:
                s = " INSERT DATA { GRAPH "+graph_name.n3()+" {" + temp_graph.serialize(format="nt") + " } } "
                print "saving ", s
            else:
                s = " INSERT DATA { " + temp_graph.serialize(format="nt") + " } "
            self.conf['rdf.graph'].update(s)

    def add_reference(self, g, reference_iri):
        """
        Add a citation to a set of statements in the database
        :param triples: A set of triples to annotate
        """
        new_statements = Graph()
        ns = self.conf['rdf.namespace']
        for statement in g:
            statement_node = self._reify(new_statements,statement)
            new_statements.add((URIRef(reference_iri), ns['asserts'], statement_node))

        self.add_statements(g + new_statements)

    #def _add_unannotated_statements(self, graph):
    # A UTC class.

    def retract_statements(self, graph):
        """
        Remove a set of statements from the database.
        :param graph: An iterable of triples
        """
        self._remove_from_store(graph)

    def add_statements(self, graph):
        """
        Add a set of statements to the database.
        Annotates the addition with uploader name, etc
        :param graph: An iterable of triples
        """
        #uri = self.conf['molecule_name'](graph.identifier)

        ns = self.conf['rdf.namespace']
        self._add_to_store(graph)

    def _reify(self,g,s):
        """
        Add a statement object to g that binds to s
        """
        n = self.conf['new_graph_uri'](s)
        g.add((n, RDF['type'], RDF['Statement']))
        g.add((n, RDF['subject'], s[0]))
        g.add((n, RDF['predicate'], s[1]))
        g.add((n, RDF['object'], s[2]))
        return n

class Data(Configure, Configureable):
    def __init__(self, conf=False):
        Configure.__init__(self)
        Configureable.__init__(self,conf)
        # We copy over all of the configuration that we were given
        self.copy(self.conf)
        self.namespace = Namespace("http://openworm.org/entities/")
        self.molecule_namespace = Namespace("http://openworm.org/entities/molecules/")
        self['nx'] = _B(self._init_networkX)
        self['rdf.namespace'] = self.namespace
        self['molecule_name'] = self._molecule_hash
        self['new_graph_uri'] = self._molecule_hash
        self._init_rdf_graph()

    @classmethod
    def open(cls,file_name):
        c = Configure.open(file_name)
        return cls(c)

    def openDatabase(self):
        self.source.open()

    def closeDatabase(self):
        self.source.close()

    def _init_rdf_graph(self):
        # Set these in case they were left out
        c = self.conf
        c['rdf.store'] = self.conf.get('rdf.store', 'default')
        c['rdf.store_conf'] = self.conf.get('rdf.store_conf', 'default')

        d = {'sqlite' : SQLiteSource(c),
                'sparql_endpoint' : SPARQLSource(c),
                'Sleepycat' : SleepyCatSource(c),
                'TriX' : TrixSource(c)}
        i = d[self.conf['rdf.source']]
        self.source = i
        self.link('semantic_net_new', 'semantic_net', 'rdf.graph')
        self['rdf.graph'] = i
        return i

    def _molecule_hash(self, data):
        return URIRef(self.molecule_namespace[hashlib.sha224(str(data)).hexdigest()])

    def _init_networkX(self):
        g = nx.DiGraph()

        # Neuron table
        csvfile = urllib2.urlopen(self.conf['neuronscsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            neurontype = ""
            # Detects neuron function
            if "sensory" in row[1].lower():
                neurontype += "sensory"
            if "motor" in row[1].lower():
                neurontype += "motor"
            if "interneuron" in row[1].lower():
                neurontype += "interneuron"
            if len(neurontype) == 0:
                neurontype = "unknown"

            if len(row[0]) > 0: # Only saves valid neuron names
                g.add_node(row[0], ntype = neurontype)

        # Connectome table
        csvfile = urllib2.urlopen(self.conf['connectomecsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            g.add_edge(row[0], row[1], weight = row[3])
            g[row[0]][row[1]]['synapse'] = row[2]
            g[row[0]][row[1]]['neurotransmitter'] = row[4]
        return g

def modification_date(filename):
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

class TrixSource(Configureable,PyOpenWorm.ConfigValue):
    """ Reads from a TriX file or if the store is more recent, from that. """
    # XXX How to write back out to this?
    i = 0
    def __init__(self, conf=False):
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def get(self):
        return self.graph

    def close(self):
        self.graph.close()

    def open(self):
        if not self.graph:
            self.graph = True
            import glob
            # Check the ages of the files. Read the more recent one.
            g0 = ConjunctiveGraph(store=self.conf['rdf.store'])
            database_store = self.conf['rdf.store_conf']
            trix_file = self.conf['trix_location']
            # store_time only works for stores that are on the local
            # machine.
            try:
                store_time = modification_date(database_store)
                # If the store is newer than the serialization
                # get the newest file in the store
                for x in glob.glob(database_store +"/*"):
                    mod = modification_date(x)
                    if store_time < mod:
                        store_time = mod
            except:
                store_time = DT.min

            trix_time = modification_date(trix_file)

            g0.open(database_store, create=True)

            if store_time > trix_time:
                # just use the store
                pass
            else:
                # delete the database and read in the new one
                # read in the serialized format
                g0.parse(trix_file,format="trix")

            self.graph = g0

        return self.graph

class SPARQLSource(Configureable,PyOpenWorm.ConfigValue):
    def __init__(self,conf=False):
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('SPARQLUpdateStore')
        g0.open(tuple(self.conf['rdf.store_conf']))
        self.graph = g0
        return self.graph

    def close(self):
        self.graph.close()

    def get(self):
        return self.graph

class SleepyCatSource(Configureable,PyOpenWorm.ConfigValue):
    def __init__(self,conf=False):
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('Sleepycat')
        g0.open(self.conf['rdf.store_conf'])
        self.graph = g0
    def close(self):
        self.graph.close()
    def get(self):
        return self.graph

class SQLiteSource(Configureable,PyOpenWorm.ConfigValue):
    def get(self):
        return self.graph
    def open(self):
        conn = sqlite3.connect(self.conf['sqldb'])
        cur = conn.cursor()

        #first step, grab all entities and add them to the graph
        n = self.conf['rdf.namespace']

        cur.execute("SELECT DISTINCT ID, Entity FROM tblentity")
        g0 = ConjunctiveGraph(self.conf['rdf.store'])
        g0.open(self.conf['rdf.store_conf'], create=True)

        for r in cur.fetchall():
            #first item is a number -- needs to be converted to a string
           first = str(r[0])
           #second item is text
           second = str(r[1])

           # This is the backbone of any RDF graph.  The unique
           # ID for each entity is encoded as a URI and every other piece of
           # knowledge about that entity is connected via triples to that URI
           # In this case, we connect the common name of that entity to the
           # root URI via the RDFS label property.
           g0.add( (n[first], RDFS.label, Literal(second)) )

        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship")

        gi = ''

        i = 0
        for r in cur.fetchall():
           #all items are numbers -- need to be converted to a string
           first = str(r[0])
           second = str(r[1])
           third = str(r[2])
           prov = str(r[3])

           ui = self.conf['molecule_name'](prov)
           gi = Graph(g0.store, ui)

           gi.add( (n[first], n[second], n[third]) )

           g0.add([ui, RDFS.label, Literal(str(i))])
           if (prov != ''):
               g0.add([ui, n[u'text_reference'], Literal(prov)])

           i = i + 1

        cur.close()
        conn.close()
        self.graph = g0

    def close(self):
        self.graph.close()
<commit_msg>Raising an error when attempting to use an un-opened graph store<commit_after>
# A consolidation of the data sources for the project
# includes:
# NetworkX!
# RDFlib!
# Other things!
#
# Works like Configure:
# Inherit from the Data class to access data of all kinds (listed above)

import sqlite3
import networkx as nx
import PyOpenWorm
from PyOpenWorm import Configureable, Configure, ConfigValue, BadConf
import hashlib
import csv
import urllib2
from rdflib import URIRef, Literal, Graph, Namespace, ConjunctiveGraph, BNode
from rdflib.namespace import RDFS,RDF
from datetime import datetime as DT
import datetime
from rdflib.namespace import XSD
from itertools import izip_longest
import os

# encapsulates some of the data all of the parts need...
class _B(ConfigValue):
    def __init__(self, f):
        self.v = False
        self.f = f

    def get(self):
        if not self.v:
            self.v = self.f()

        return self.v
    def invalidate(self):
        self.v = False

ZERO = datetime.timedelta(0)
class _UTC(datetime.tzinfo):
    """UTC"""

    def utcoffset(self, dt):
        return ZERO

    def tzname(self, dt):
        return "UTC"

    def dst(self, dt):
        return ZERO
utc = _UTC()

class _Z(ConfigValue):
    def __init__(self, c, n):
        self.n = n
    def get(self):
        return c[n]

propertyTypes = {"send" : 'http://openworm.org/entities/356',
        "Neuropeptide" : 'http://openworm.org/entities/354',
        "Receptor" : 'http://openworm.org/entities/361',
        "is a" : 'http://openworm.org/entities/1515',
        "neuromuscular junction" : 'http://openworm.org/entities/1516',
        "Innexin" : 'http://openworm.org/entities/355',
        "Neurotransmitter" : 'http://openworm.org/entities/313',
        "gap junction" : 'http://openworm.org/entities/357'}

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    while True:
        l = []
        try:
            for x in args:
                l.append(next(x))
        except:
            pass
        yield l
        if len(l) < n:
            break

class DataUser(Configureable):
    def __init__(self, conf = False):
        Configureable.__init__(self, conf=conf)
        if not isinstance(self.conf,Data):
            raise BadConf(self)

    @property
    def rdf(self):
        return self.conf['rdf.graph']

    def _remove_from_store(self, g):
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            s = " DELETE DATA {" + temp_graph.serialize(format="nt") + " } "
            self.conf['rdf.graph'].update(s)

    def _add_to_store(self, g, graph_name=False):
        for group in grouper(g, 1000):
            temp_graph = Graph()
            for x in group:
                if x is not None:
                    temp_graph.add(x)
                else:
                    break
            if graph_name:
                s = " INSERT DATA { GRAPH "+graph_name.n3()+" {" + temp_graph.serialize(format="nt") + " } } "
            else:
                s = " INSERT DATA { " + temp_graph.serialize(format="nt") + " } "
            self.conf['rdf.graph'].update(s)

    def add_reference(self, g, reference_iri):
        """
        Add a citation to a set of statements in the database
        :param triples: A set of triples to annotate
        """
        new_statements = Graph()
        ns = self.conf['rdf.namespace']
        for statement in g:
            statement_node = self._reify(new_statements,statement)
            new_statements.add((URIRef(reference_iri), ns['asserts'], statement_node))

        self.add_statements(g + new_statements)

    #def _add_unannotated_statements(self, graph):
    # A UTC class.

    def retract_statements(self, graph):
        """
        Remove a set of statements from the database.
        :param graph: An iterable of triples
        """
        self._remove_from_store(graph)

    def add_statements(self, graph):
        """
        Add a set of statements to the database.
        Annotates the addition with uploader name, etc
        :param graph: An iterable of triples
        """
        #uri = self.conf['molecule_name'](graph.identifier)

        ns = self.conf['rdf.namespace']
        self._add_to_store(graph)

    def _reify(self,g,s):
        """
        Add a statement object to g that binds to s
        """
        n = self.conf['new_graph_uri'](s)
        g.add((n, RDF['type'], RDF['Statement']))
        g.add((n, RDF['subject'], s[0]))
        g.add((n, RDF['predicate'], s[1]))
        g.add((n, RDF['object'], s[2]))
        return n

class Data(Configure, Configureable):
    def __init__(self, conf=False):
        Configure.__init__(self)
        Configureable.__init__(self,conf)
        # We copy over all of the configuration that we were given
        self.copy(self.conf)
        self.namespace = Namespace("http://openworm.org/entities/")
        self.molecule_namespace = Namespace("http://openworm.org/entities/molecules/")
        self['nx'] = _B(self._init_networkX)
        self['rdf.namespace'] = self.namespace
        self['molecule_name'] = self._molecule_hash
        self['new_graph_uri'] = self._molecule_hash
        self._init_rdf_graph()

    @classmethod
    def open(cls,file_name):
        c = Configure.open(file_name)
        return cls(c)

    def openDatabase(self):
        self.source.open()

    def closeDatabase(self):
        self.source.close()

    def _init_rdf_graph(self):
        # Set these in case they were left out
        c = self.conf
        c['rdf.store'] = self.conf.get('rdf.store', 'default')
        c['rdf.store_conf'] = self.conf.get('rdf.store_conf', 'default')

        d = {'sqlite' : SQLiteSource(c),
                'sparql_endpoint' : SPARQLSource(c),
                'Sleepycat' : SleepyCatSource(c),
                'TriX' : TrixSource(c)}
        i = d[self.conf['rdf.source']]
        self.source = i
        self.link('semantic_net_new', 'semantic_net', 'rdf.graph')
        self['rdf.graph'] = i
        return i

    def _molecule_hash(self, data):
        return URIRef(self.molecule_namespace[hashlib.sha224(str(data)).hexdigest()])

    def _init_networkX(self):
        g = nx.DiGraph()

        # Neuron table
        csvfile = urllib2.urlopen(self.conf['neuronscsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            neurontype = ""
            # Detects neuron function
            if "sensory" in row[1].lower():
                neurontype += "sensory"
            if "motor" in row[1].lower():
                neurontype += "motor"
            if "interneuron" in row[1].lower():
                neurontype += "interneuron"
            if len(neurontype) == 0:
                neurontype = "unknown"

            if len(row[0]) > 0: # Only saves valid neuron names
                g.add_node(row[0], ntype = neurontype)

        # Connectome table
        csvfile = urllib2.urlopen(self.conf['connectomecsv'])

        reader = csv.reader(csvfile, delimiter=';', quotechar='|')
        for row in reader:
            g.add_edge(row[0], row[1], weight = row[3])
            g[row[0]][row[1]]['synapse'] = row[2]
            g[row[0]][row[1]]['neurotransmitter'] = row[4]
        return g

def modification_date(filename):
    t = os.path.getmtime(filename)
    return datetime.datetime.fromtimestamp(t)

class RDFSource(Configureable,PyOpenWorm.ConfigValue):
    def __init__(self, conf=False):
        Configureable.__init__(self, conf=conf)
        self.graph = False

    def get(self):
        if not self.graph:
            raise Exception("Must call openDatabase on Data object before using the database")
        return self.graph

    def close(self):
        self.graph.close()

class TrixSource(RDFSource):
    """ Reads from a TriX file or if the store is more recent, from that. """
    # XXX How to write back out to this?
    i = 0

    def open(self):
        if not self.graph:
            self.graph = True
            import glob
            # Check the ages of the files. Read the more recent one.
            g0 = ConjunctiveGraph(store=self.conf['rdf.store'])
            database_store = self.conf['rdf.store_conf']
            trix_file = self.conf['trix_location']
            # store_time only works for stores that are on the local
            # machine.
            try:
                store_time = modification_date(database_store)
                # If the store is newer than the serialization
                # get the newest file in the store
                for x in glob.glob(database_store +"/*"):
                    mod = modification_date(x)
                    if store_time < mod:
                        store_time = mod
            except:
                store_time = DT.min

            trix_time = modification_date(trix_file)

            g0.open(database_store, create=True)

            if store_time > trix_time:
                # just use the store
                pass
            else:
                # delete the database and read in the new one
                # read in the serialized format
                g0.parse(trix_file,format="trix")

            self.graph = g0

        return self.graph

class SPARQLSource(RDFSource):
    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('SPARQLUpdateStore')
        g0.open(tuple(self.conf['rdf.store_conf']))
        self.graph = g0
        return self.graph

class SleepyCatSource(RDFSource):
    def open(self):
        # XXX: If we have a source that's read only, should we need to set the store separately??
        g0 = ConjunctiveGraph('Sleepycat')
        g0.open(self.conf['rdf.store_conf'])
        self.graph = g0

class SQLiteSource(RDFSource):
    def open(self):
        conn = sqlite3.connect(self.conf['sqldb'])
        cur = conn.cursor()

        #first step, grab all entities and add them to the graph
        n = self.conf['rdf.namespace']

        cur.execute("SELECT DISTINCT ID, Entity FROM tblentity")
        g0 = ConjunctiveGraph(self.conf['rdf.store'])
        g0.open(self.conf['rdf.store_conf'], create=True)

        for r in cur.fetchall():
            #first item is a number -- needs to be converted to a string
           first = str(r[0])
           #second item is text
           second = str(r[1])

           # This is the backbone of any RDF graph.  The unique
           # ID for each entity is encoded as a URI and every other piece of
           # knowledge about that entity is connected via triples to that URI
           # In this case, we connect the common name of that entity to the
           # root URI via the RDFS label property.
           g0.add( (n[first], RDFS.label, Literal(second)) )

        #second step, get the relationships between them and add them to the graph
        cur.execute("SELECT DISTINCT EnID1, Relation, EnID2, Citations FROM tblrelationship")

        gi = ''

        i = 0
        for r in cur.fetchall():
           #all items are numbers -- need to be converted to a string
           first = str(r[0])
           second = str(r[1])
           third = str(r[2])
           prov = str(r[3])

           ui = self.conf['molecule_name'](prov)
           gi = Graph(g0.store, ui)

           gi.add( (n[first], n[second], n[third]) )

           g0.add([ui, RDFS.label, Literal(str(i))])
           if (prov != ''):
               g0.add([ui, n[u'text_reference'], Literal(prov)])

           i = i + 1

        cur.close()
        conn.close()
        self.graph = g0
<|endoftext|>


<commit_before>import uuid
import traceback

import protocol.Sirikata_pb2 as pbSiri
import protocol.Persistence_pb2 as pbPer
import protocol.MessageHeader_pb2 as pbHead

from Sirikata.Runtime import HostedObject
print dir(HostedObject)

import util

class exampleclass:
    def __init__(self):
        self.val=0
    def func(self,otherval):
        self.val+=otherval
        print self.val
        return self.val;
    def reallyProcessRPC(self,serialheader,name,serialarg):
        print "Got an RPC named",name
        header = pbHead.Header()
        header.ParseFromString(util.fromByteArray(serialheader))
        if name == "RetObj":
            retobj = pbSiri.RetObj()
            #print repr(util.fromByteArray(serialarg))
            try:
                retobj.ParseFromString(util.fromByteArray(serialarg))
            except:
                pass
            self.objid = retobj.object_reference
            print "sendprox1"
            self.spaceid = header.source_space

            print util.tupleToUUID(self.spaceid)
            self.sendNewProx()
            self.setPosition(angular_speed=1,axis=(0,1,0))
        elif name == "ProxCall":
            proxcall = pbSiri.ProxCall()
            proxcall.ParseFromString(util.fromByteArray(serialarg))
            objRef = util.tupleToUUID(proxcall.proximate_object)
            if proxcall.proximity_event == pbSiri.ProxCall.ENTERED_PROXIMITY:
                myhdr = pbHead.Header()
                myhdr.destination_space = self.spaceid
                myhdr.destination_object = self.objid
                dbQuery = util.PersistenceRead(self.sawAnotherObject)
                field = dbQuery.reads.add()
                field.field_name = 'Name'
                dbQuery.send(HostedObject, myhdr)
            if proxcall.proximity_event == pbSiri.ProxCall.EXITED_PROXIMITY:
                pass
    def sawAnotherObject(self,persistence,header,retstatus):
        print "PY: sawAnotherObject called"
        if header.HasField('return_status') or retstatus:
            return
        uuid = util.tupleToUUID(header.source_object)
        myName = ""
        for field in persistence:
            if field.field_name == 'Name':
                if field.HasField('data'):
                    myName = field.data
        print "Object",uuid,"has name",myName
    def processRPC(self,header,name,arg):
        try:
            self.reallyProcessRPC(header,name,arg)
        except:
            print "Error processing RPC",name
            traceback.print_exc()
    def setPosition(self,position=None,orientation=None,velocity=None,angular_speed=None,axis=None,force=False):
        objloc = pbSiri.ObjLoc()
        if position is not None:
            for i in range(3):
                objloc.position.append(position[i])
        if velocity is not None:
            for i in range(3):
                objloc.velocity.append(velocity[i])
        if orientation is not None:
            total = 0
            for i in range(4):
                total += orientation[i]*orientation[i]
            total = total**.5
            for i in range(3):
                objloc.orientation.append(orientation[i]/total)
        if angular_speed is not None:
            objloc.angular_speed = angular_speed
        if axis is not None:
            total = 0
            for i in range(3):
                total += axis[i]*axis[i]
            total = total**.5
            for i in range(2):
                objloc.rotational_axis.append(axis[i]/total)
        if force:
            objloc.update_flags = pbSiri.ObjLoc.FORCE
        body = pbSiri.MessageBody()
        body.message_names.append("SetLoc")
        body.message_arguments.append(objloc.SerializeToString())
        header = pbHead.Header()
        header.destination_space = self.spaceid
        header.destination_object = self.objid
        HostedObject.SendMessage(util.toByteArray(header.SerializeToString()+body.SerializeToString()))
    def sendNewProx(self):
        print "sendprox2"
        try:
            print "sendprox3"
            body = pbSiri.MessageBody()
            prox = pbSiri.NewProxQuery()
            prox.query_id = 123
            print "sendprox4"
            prox.max_radius = 1000.0
            body.message_names.append("NewProxQuery")
            body.message_arguments.append(prox.SerializeToString())
            header = pbHead.Header()
            print "sendprox5"
            header.destination_space = self.spaceid;
            header.destination_object = util.tupleFromUUID(uuid.UUID(int=0))
            header.destination_port = 3 # libcore/src/util/KnownServices.hpp
            headerstr = header.SerializeToString()
            bodystr = body.SerializeToString()
            HostedObject.SendMessage(util.toByteArray(headerstr+bodystr))
        except:
            print "ERORR"
            traceback.print_exc()

    def processMessage(self,header,body):
        print "Got a message"
    def tick(self,tim):
        x=str(tim)
        print "Current time is "+x;
        #HostedObject.SendMessage(tuple(Byte(ord(c)) for c in x));# this seems to get into hosted object...but fails due to bad encoding
<commit_msg>fixed bug with persistence structure in python not having right member name<commit_after>import uuid
import traceback

import protocol.Sirikata_pb2 as pbSiri
import protocol.Persistence_pb2 as pbPer
import protocol.MessageHeader_pb2 as pbHead

from Sirikata.Runtime import HostedObject
print dir(HostedObject)

import util

class exampleclass:
    def __init__(self):
        self.val=0
    def func(self,otherval):
        self.val+=otherval
        print self.val
        return self.val;
    def reallyProcessRPC(self,serialheader,name,serialarg):
        print "Got an RPC named",name
        header = pbHead.Header()
        header.ParseFromString(util.fromByteArray(serialheader))
        if name == "RetObj":
            retobj = pbSiri.RetObj()
            #print repr(util.fromByteArray(serialarg))
            try:
                retobj.ParseFromString(util.fromByteArray(serialarg))
            except:
                pass
            self.objid = retobj.object_reference
            print "sendprox1"
            self.spaceid = header.source_space

            print util.tupleToUUID(self.spaceid)
            self.sendNewProx()
            self.setPosition(angular_speed=1,axis=(0,1,0))
        elif name == "ProxCall":
            proxcall = pbSiri.ProxCall()
            proxcall.ParseFromString(util.fromByteArray(serialarg))
            objRef = util.tupleToUUID(proxcall.proximate_object)
            if proxcall.proximity_event == pbSiri.ProxCall.ENTERED_PROXIMITY:
                myhdr = pbHead.Header()
                myhdr.destination_space = self.spaceid
                myhdr.destination_object = self.objid
                dbQuery = util.PersistenceRead(self.sawAnotherObject)
                field = dbQuery.reads.add()
                field.field_name = 'Name'
                dbQuery.send(HostedObject, myhdr)
            if proxcall.proximity_event == pbSiri.ProxCall.EXITED_PROXIMITY:
                pass
    def sawAnotherObject(self,persistence,header,retstatus):
        print "PY: sawAnotherObject called"
        if header.HasField('return_status') or retstatus:
            return
        uuid = util.tupleToUUID(header.source_object)
        myName = ""
        for field in persistence.reads:
            print field
            if field.field_name == 'Name':
                if field.HasField('data'):
                    myName = field.data
        print "Object",uuid,"has name",myName
    def processRPC(self,header,name,arg):
        try:
            self.reallyProcessRPC(header,name,arg)
        except:
            print "Error processing RPC",name
            traceback.print_exc()
    def setPosition(self,position=None,orientation=None,velocity=None,angular_speed=None,axis=None,force=False):
        objloc = pbSiri.ObjLoc()
        if position is not None:
            for i in range(3):
                objloc.position.append(position[i])
        if velocity is not None:
            for i in range(3):
                objloc.velocity.append(velocity[i])
        if orientation is not None:
            total = 0
            for i in range(4):
                total += orientation[i]*orientation[i]
            total = total**.5
            for i in range(3):
                objloc.orientation.append(orientation[i]/total)
        if angular_speed is not None:
            objloc.angular_speed = angular_speed
        if axis is not None:
            total = 0
            for i in range(3):
                total += axis[i]*axis[i]
            total = total**.5
            for i in range(2):
                objloc.rotational_axis.append(axis[i]/total)
        if force:
            objloc.update_flags = pbSiri.ObjLoc.FORCE
        body = pbSiri.MessageBody()
        body.message_names.append("SetLoc")
        body.message_arguments.append(objloc.SerializeToString())
        header = pbHead.Header()
        header.destination_space = self.spaceid
        header.destination_object = self.objid
        HostedObject.SendMessage(util.toByteArray(header.SerializeToString()+body.SerializeToString()))
    def sendNewProx(self):
        print "sendprox2"
        try:
            print "sendprox3"
            body = pbSiri.MessageBody()
            prox = pbSiri.NewProxQuery()
            prox.query_id = 123
            print "sendprox4"
            prox.max_radius = 1000.0
            body.message_names.append("NewProxQuery")
            body.message_arguments.append(prox.SerializeToString())
            header = pbHead.Header()
            print "sendprox5"
            header.destination_space = self.spaceid;
            header.destination_object = util.tupleFromUUID(uuid.UUID(int=0))
            header.destination_port = 3 # libcore/src/util/KnownServices.hpp
            headerstr = header.SerializeToString()
            bodystr = body.SerializeToString()
            HostedObject.SendMessage(util.toByteArray(headerstr+bodystr))
        except:
            print "ERORR"
            traceback.print_exc()

    def processMessage(self,header,body):
        print "Got a message"
    def tick(self,tim):
        x=str(tim)
        print "Current time is "+x;
        #HostedObject.SendMessage(tuple(Byte(ord(c)) for c in x));# this seems to get into hosted object...but fails due to bad encoding
<|endoftext|>


<commit_before># Copyright 2014-2018 The ODL contributors
#
# This file is part of ODL
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Tests for the d.c. solvers."""

from __future__ import division
import odl
from odl.solvers import (dca, prox_dca, doubleprox_dc)
import numpy as np
import pytest

# Places for the accepted error when comparing results
HIGH_ACCURACY = 8
LOW_ACCURACY = 4

def test_dca():
    """Test the dca method for the following simple problem:

    Let a > 0, and let b be a real number. Solve

    min_x a/2 (x - b)^2 - |x|

    For finding possible (local) solutions, we consider several cases:
    * x > 0 ==> ∂|.|(x) = 1, i.e., a necessary optimality condition is
      0 = a (x - b) - 1 ==> x = b + 1/a. This only happens if b > -1/a.
    * x < 0 ==> ∂|.|(x) = -1, i.e., a necessary optimality condition is
      0 = a (x - b) + 1 ==> x = b - 1/a. This only happens if b < 1/a.
    * x = 0 ==> ∂|.|(x) = [-1, 1], i.e., a necessary optimality condition is
      0 = a (x - b) + [-1, 1] ==> b - 1/a <= x = 0 <= b + 1/a.

    To summarize, we might find the following solution sets:
    * {b - 1/a} if b < -1/a,
    * {-2/a, 0} if b = -1/a,
    * {b - 1/a, 0, b + 1/a} if -1/a < b < 1/a,
    * {0, 2/a} if b = 1/a,
    * {b + 1/a} if b > 1/a.
    """

    # Set the problem parameters
    a = 0.5
    b = 0.5
    space = odl.rn(1)
    g = a / 2 * odl.solvers.L2NormSquared(space).translated(b)
    h = odl.solvers.L1Norm(space)
    niter = 50

    # Set up some space elements for the solvers to use
    x = space.element(-0.5)
    x_dca = x.copy()
    x_prox_dca = x.copy()
    x_doubleprox = x.copy()

    # Some additional parameters for some of the solvers
    phi = odl.solvers.ZeroFunctional(space)
    y = space.element(3)
    gamma = 1
    mu = 1
    K = odl.IdentityOperator(space)

    dca(x_dca, g, h, niter)
    prox_dca(x_prox_dca, g, h, niter, gamma)
    doubleprox_dc(x_doubleprox, y, g, h, phi, K, niter, gamma, mu)
    expected = np.asarray([b - 1 / a, 0, b + 1 / a])

    dist_dca = np.min(np.abs(expected - float(x_dca)))
    dist_prox_cda = np.min(np.abs(expected - float(x_prox_dca)))
    dist_prox_doubleprox = np.min(np.abs(expected - float(x_doubleprox)))

    assert dist_dca == pytest.approx(0, abs=1e-6)
    assert dist_prox_cda == pytest.approx(0, abs=1e-6)
    assert dist_prox_doubleprox == pytest.approx(0, abs=1e-6)
<commit_msg>BUG: Add a unicode marker to difference_convex_test.py for Python 2.<commit_after># coding=utf-8

# Copyright 2014-2018 The ODL contributors
#
# This file is part of ODL
#
# This Source Code Form is subject to the terms of the Mozilla Public License,
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
# obtain one at https://mozilla.org/MPL/2.0/.

"""Tests for the d.c. solvers."""

from __future__ import division
import odl
from odl.solvers import (dca, prox_dca, doubleprox_dc)
import numpy as np
import pytest

# Places for the accepted error when comparing results
HIGH_ACCURACY = 8
LOW_ACCURACY = 4

def test_dca():
    """Test the dca method for the following simple problem:

    Let a > 0, and let b be a real number. Solve

    min_x a/2 (x - b)^2 - |x|

    For finding possible (local) solutions, we consider several cases:
    * x > 0 ==> ∂|.|(x) = 1, i.e., a necessary optimality condition is
      0 = a (x - b) - 1 ==> x = b + 1/a. This only happens if b > -1/a.
    * x < 0 ==> ∂|.|(x) = -1, i.e., a necessary optimality condition is
      0 = a (x - b) + 1 ==> x = b - 1/a. This only happens if b < 1/a.
    * x = 0 ==> ∂|.|(x) = [-1, 1], i.e., a necessary optimality condition is
      0 = a (x - b) + [-1, 1] ==> b - 1/a <= x = 0 <= b + 1/a.

    To summarize, we might find the following solution sets:
    * {b - 1/a} if b < -1/a,
    * {-2/a, 0} if b = -1/a,
    * {b - 1/a, 0, b + 1/a} if -1/a < b < 1/a,
    * {0, 2/a} if b = 1/a,
    * {b + 1/a} if b > 1/a.
    """

    # Set the problem parameters
    a = 0.5
    b = 0.5
    space = odl.rn(1)
    g = a / 2 * odl.solvers.L2NormSquared(space).translated(b)
    h = odl.solvers.L1Norm(space)
    niter = 50

    # Set up some space elements for the solvers to use
    x = space.element(-0.5)
    x_dca = x.copy()
    x_prox_dca = x.copy()
    x_doubleprox = x.copy()

    # Some additional parameters for some of the solvers
    phi = odl.solvers.ZeroFunctional(space)
    y = space.element(3)
    gamma = 1
    mu = 1
    K = odl.IdentityOperator(space)

    dca(x_dca, g, h, niter)
    prox_dca(x_prox_dca, g, h, niter, gamma)
    doubleprox_dc(x_doubleprox, y, g, h, phi, K, niter, gamma, mu)
    expected = np.asarray([b - 1 / a, 0, b + 1 / a])

    dist_dca = np.min(np.abs(expected - float(x_dca)))
    dist_prox_cda = np.min(np.abs(expected - float(x_prox_dca)))
    dist_prox_doubleprox = np.min(np.abs(expected - float(x_doubleprox)))

    assert dist_dca == pytest.approx(0, abs=1e-6)
    assert dist_prox_cda == pytest.approx(0, abs=1e-6)
    assert dist_prox_doubleprox == pytest.approx(0, abs=1e-6)
<|endoftext|>


<commit_before>__all__ = ['help_me', 'RecipeParser', 'main']<commit_msg>PEP-8<commit_after>__all__ = ['help_me', 'RecipeParser', 'main']
<|endoftext|>