<commit_before>import sys
import time
import logging
logging.basicConfig(level=logging.DEBUG)

from redis import StrictRedis
from rq import Queue
from apscheduler.schedulers.blocking import BlockingScheduler

sys.path.append('/d1lod')
from d1lod import jobs

conn = StrictRedis(host='redis', port='6379')
q = Queue(connection=conn)

sched = BlockingScheduler()

@sched.scheduled_job('interval', minutes=1)
def update_job():
    q.enqueue(jobs.update_graph)

@sched.scheduled_job('interval', hours=1)
def stats_job():
    q.enqueue(jobs.calculate_stats)

@sched.scheduled_job('interval', hours=1)
def export_job():
    q.enqueue(jobs.export_graph)

@sched.scheduled_job('interval', minutes=1)
def print_jobs_job():
    sched.print_jobs()

time.sleep(10)
sched.start()
<commit_msg>Run export every minute<commit_after>import sys
import time
import logging
logging.basicConfig(level=logging.DEBUG)

from redis import StrictRedis
from rq import Queue
from apscheduler.schedulers.blocking import BlockingScheduler

sys.path.append('/d1lod')
from d1lod import jobs

conn = StrictRedis(host='redis', port='6379')
q = Queue(connection=conn)

sched = BlockingScheduler()

@sched.scheduled_job('interval', minutes=1)
def update_job():
    q.enqueue(jobs.update_graph)

@sched.scheduled_job('interval', hours=1)
def stats_job():
    q.enqueue(jobs.calculate_stats)

@sched.scheduled_job('interval', minutes=1)
def export_job():
    q.enqueue(jobs.export_graph)

@sched.scheduled_job('interval', minutes=1)
def print_jobs_job():
    sched.print_jobs()

time.sleep(10)
sched.start()
<|endoftext|>


<commit_before>"""
Example settings module.

This should be copied as `settings.py` and the values modified there.

That file is ignored by the repo, since it will contain environment
specific and sensitive information (like passwords).
"""

import logging
import os
import re

import yaml

from imap import EmailCheckError, EmailServer
from postgresql import DatabaseServer

# If this is set to a valid path, all CSV files extracted from emails will be
# stored in sub-folders within it.
CSV_FOLDER = os.getcwd()

SETTINGS_YAML_PATH = os.path.join(os.getcwd(), 'settings.yaml')

LOGGING_FORMAT = '''
- file: %(pathname)s
  level: %(levelname)s
  line: %(lineno)s
  message: |
    %(message)s
  time: %(asctime)s
'''.strip()

LOGGING_LEVEL = logging.DEBUG

def get_csv_file_types():
	csv_file_types = None

	with open(SETTINGS_YAML_PATH) as r:
		csv_file_types = yaml.load(r)

	return csv_file_types

def get_database_client():
	con = 'my_username/my_password@database.example.com:5432/my_database'
	return DatabaseServer(con)

def get_email_client():
	return EmailServer('mail.example.com', 'my_username', 'my_password')
<commit_msg>Change file name to logger name in example settings<commit_after>"""
Example settings module.

This should be copied as `settings.py` and the values modified there.

That file is ignored by the repo, since it will contain environment
specific and sensitive information (like passwords).
"""

import logging
import os
import re

import yaml

from imap import EmailCheckError, EmailServer
from postgresql import DatabaseServer

# If this is set to a valid path, all CSV files extracted from emails will be
# stored in sub-folders within it.
CSV_FOLDER = os.getcwd()

SETTINGS_YAML_PATH = os.path.join(os.getcwd(), 'settings.yaml')

#- file: %(pathname)s
#  function: %(funcName)s
LOGGING_FORMAT = '''
- level: %(levelname)s
  line: %(lineno)s
  logger: %(name)s
  message: |
    %(message)s
  time: %(asctime)s
'''.strip()

LOGGING_LEVEL = logging.DEBUG

def get_csv_file_types():
	csv_file_types = None

	with open(SETTINGS_YAML_PATH) as r:
		csv_file_types = yaml.load(r)

	return csv_file_types

def get_database_client():
	con = 'my_username/my_password@database.example.com:5432/my_database'
	return DatabaseServer(con)

def get_email_client():
	return EmailServer('mail.example.com', 'my_username', 'my_password')
<|endoftext|>


<commit_before>import sublime
import sublime_plugin
import re
import os.path

pattern_anchor = re.compile(r'\[.+?\]')
pattern_endspace = re.compile(r' *?\z')

pattern_h1_h2_equal_dash = "^.*?(?:(?:\r\n)|\n|\r)(?:-+|=+)$"

#TOCTAG_START = "<!-- MarkdownTOC depth=2 -->"
TOCTAG_END = "<!-- /MarkdownTOC -->"

class MarkdowntocInsert(sublime_plugin.TextCommand):

    def run(self, edit):

        if not self.find_tag_and_insert(edit):
            sels = self.view.sel()
            for sel in sels:
                default_depth = self.get_default_depth()

                # add TOCTAG
                toc = "<!-- MarkdownTOC depth=" + \
                    str(default_depth) + " -->\n"
                toc += "\n"
                toc += self.get_toc(default_depth, sel.end())
                toc += "\n"
                toc += TOCTAG_END + "\n"

                self.view.insert(edit, sel.begin(), toc)

        # TODO: process to add another toc when tag exists

    def get_toc_open_tag(self):
        search_results = self.view.find_all(
            "^<!-- MarkdownTOC .*-->\n",
            sublime.IGNORECASE)
        search_results = self.remove_items_in_codeblock(search_results)

        toc_open_tags = []
        for toc_open in search_results:
            if 0 < len(toc_open):

                tag_str = self.view.substr(toc_open)

                depth = re.search("depth=(\w+)", tag_str)
                depth_val = self.get_default_depth()
                if depth != None:
                    depth_val = int(depth.group(1))

                al = re.search(" autolink ", tag_str)
                autolink_val = True if al != None else False

                toc_open_tag = {
                    "region":   toc_open,
                    "depth":    depth_val,
                    "autolink": autolink_val
                }
                toc_open_tags.append(toc_open_tag)

        return toc_open_tags

    def get_toc_close_tag(self, start):
        close_tags = self.view.find_all(
            "^" + TOCTAG_END + "\n")
        close_tags = self.remove_items_in_codeblock(close_tags)
        for close_tag in close_tags:
            if start < close_tag.begin():
                return close_tag

    # Search MarkdownTOC comments in document
    def find_tag_and_insert(self, edit):
        toc_starts = self.get_toc_open_tag()
        for dic in toc_starts:

            toc_start = dic["region"]
            depth     = dic["depth"]
            autolink  = dic["autolink"]

            if 0 < len(toc_start):

                toc_close = self.get_toc_close_tag(toc_start.end())

                if toc_close:
                    toc = self.get_toc(depth, autolink, toc_close.end())
                    tocRegion = sublime.Region(
                        toc_start.end(), toc_close.begin())
                    if toc:
                        self.view.replace(edit, tocRegion, "\n" + toc + "\n")
                        log('refresh TOC content')
                        return True
                    else:
                        self.view.replace(edit, tocRegion, "\n")
                        log('TOC is empty')
                        return False
        log('cannot find TOC tags')
        return False

    # TODO: add "end" parameter
    def get_toc(self, depth, autolink, begin):

        # Search headings in docment
        if depth == 0:
            pattern_hash = "^#+?[^#]"
        else:
            pattern_hash = "^#{1," + str(depth) + "}[^#]"
        headings = self.view.find_all(
            "%s|%s" % (pattern_h1_h2_equal_dash, pattern_hash))

        headings = self.remove_items_in_codeblock(headings)

        if len(headings) < 1:
            return False

        items = []  # [[headingNum,text],...]
        for heading in headings:
            if begin < heading.end():
                lines = self.view.lines(heading)
                if len(lines) == 1:
                    # handle hash headings, ### chapter 1
                    r = sublime.Region(
                        heading.end(), self.view.line(heading).end())
                    heading_text = self.view.substr(r)
                    heading_num = heading.size() - 1
                    items.append([heading_num, heading_text])
                elif len(lines) == 2:
                    # handle - or + headings, Title 1==== section1----
                    heading_text = self.view.substr(lines[0])
                    if heading_text.strip():
                        heading_num = 1 if (
                            self.view.substr(lines[1])[0] == '=') else 2
                        items.append([heading_num, heading_text])

        if len(items) < 1:
            return

        # Shape TOC  ------------------
        items = format(items)

        # Create TOC  ------------------
        toc = ''
        id_texts = []
        for item in items:
            heading_num = item[0] - 1
            heading_text = item[1].rstrip()

            # add indent by heading_num
            for i in range(heading_num):
                toc += '\t'

            # Handling anchors ("Reference-style links")
            matchObj = pattern_anchor.search(heading_text)
            if matchObj:
                only_text = heading_text[0:matchObj.start()]
                only_text = only_text.rstrip()
                id_text = matchObj.group().replace('[','').replace(']','')
                toc += '- [' + only_text + '](#' + id_text + ')\n'
            elif autolink:
                id_text = remove_reserved_chars(heading_text.lower().replace(" ", "-"))
                n = id_texts.count(id_text)
                if 0 < n:
                    id_text += '-' + str(n)
                id_texts.append(id_text)
                toc += '- [' + heading_text + '](#' + id_text + ')\n'
            else:
                toc += '- ' + heading_text + '\n'

        return toc

    def get_default_depth(self):
        setting_file = 'MarkdownTOC.sublime-settings'
        settings = sublime.load_settings(setting_file)
        default_depth = settings.get('default_depth')
        if default_depth is None:
            default_depth = 2
            # Save "Settings - Default"
            settings.set('default_depth', default_depth)
            sublime.save_settings(setting_file)
        return default_depth

    def remove_items_in_codeblock(self, items):

        codeblocks = self.view.find_all("^`{3,}[^`]*$")
        codeblockAreas = [] # [[area_begin, area_end], ..]
        i = 0
        while i < len(codeblocks)-1:
            area_begin = codeblocks[i].begin()
            area_end   = codeblocks[i+1].begin()
            if area_begin and area_end:
                codeblockAreas.append([area_begin, area_end])
            i += 2

        items = [h for h in items if is_out_of_areas(h.begin(), codeblockAreas)]
        return items

def remove_reserved_chars(str):
    # Percent-encoding reserved characters
    delete = {
        ord(u"!"): None,
        ord(u"#"): None,
        ord(u"$"): None,
        ord(u"&"): None,
        ord(u"'"): None,
        ord(u"("): None,
        ord(u")"): None,
        ord(u"*"): None,
        ord(u"+"): None,
        ord(u","): None,
        ord(u"/"): None,
        ord(u":"): None,
        ord(u";"): None,
        ord(u"="): None,
        ord(u"?"): None,
        ord(u"@"): None,
        ord(u"["): None,
        ord(u"]"): None,
        ord(u"`"): None
    }
    return str.translate(delete)

def is_out_of_areas(num, areas):
    for area in areas:
        if area[0] < num and num < area[1]:
            return False
    return True

def format(items):
    headings = []
    for item in items:
        headings.append(item[0])
    # ----------

    # set root to 1
    min_heading = min(headings)
    if 1 < min_heading:
        for i, item in enumerate(headings):
            headings[i] -= min_heading - 1
    headings[0] = 1  # first item must be 1

    # minimize "jump width"
    for i, item in enumerate(headings):
        if 0 < i and 1 < item - headings[i - 1]:
            before = headings[i]
            after = headings[i - 1] + 1
            headings[i] = after
            for n in range(i + 1, len(headings)):
                if(headings[n] == before):
                    headings[n] = after
                else:
                    break

    # ----------
    for i, item in enumerate(items):
        item[0] = headings[i]
    return items

def log(arg):
    sublime.status_message(arg)
    print(arg)

# Search and refresh if it's exist

class MarkdowntocUpdate(MarkdowntocInsert):

    def run(self, edit):
        MarkdowntocInsert.find_tag_and_insert(self, edit)

class AutoRunner(sublime_plugin.EventListener):

    def on_pre_save(self, view):
        # limit scope
        root, ext = os.path.splitext(view.file_name())
        ext = ext.lower()
        if ext in [".md", ".markdown", ".mdown", ".mdwn", ".mkdn", ".mkd", ".mark"]:
            view.run_command('markdowntoc_update')
<commit_msg>refix: same name issue #13<commit_after>import sublime
import sublime_plugin
import re
import os.path

pattern_anchor = re.compile(r'\[.+?\]')
pattern_endspace = re.compile(r' *?\z')

pattern_h1_h2_equal_dash = "^.*?(?:(?:\r\n)|\n|\r)(?:-+|=+)$"

#TOCTAG_START = "<!-- MarkdownTOC depth=2 -->"
TOCTAG_END = "<!-- /MarkdownTOC -->"

class MarkdowntocInsert(sublime_plugin.TextCommand):

    def run(self, edit):

        if not self.find_tag_and_insert(edit):
            sels = self.view.sel()
            for sel in sels:
                default_depth = self.get_default_depth()

                # add TOCTAG
                toc = "<!-- MarkdownTOC depth=" + \
                    str(default_depth) + " -->\n"
                toc += "\n"
                toc += self.get_toc(default_depth, sel.end())
                toc += "\n"
                toc += TOCTAG_END + "\n"

                self.view.insert(edit, sel.begin(), toc)

        # TODO: process to add another toc when tag exists

    def get_toc_open_tag(self):
        search_results = self.view.find_all(
            "^<!-- MarkdownTOC .*-->\n",
            sublime.IGNORECASE)
        search_results = self.remove_items_in_codeblock(search_results)

        toc_open_tags = []
        for toc_open in search_results:
            if 0 < len(toc_open):

                tag_str = self.view.substr(toc_open)

                depth = re.search("depth=(\w+)", tag_str)
                depth_val = self.get_default_depth()
                if depth != None:
                    depth_val = int(depth.group(1))

                al = re.search(" autolink ", tag_str)
                autolink_val = True if al != None else False

                toc_open_tag = {
                    "region":   toc_open,
                    "depth":    depth_val,
                    "autolink": autolink_val
                }
                toc_open_tags.append(toc_open_tag)

        return toc_open_tags

    def get_toc_close_tag(self, start):
        close_tags = self.view.find_all(
            "^" + TOCTAG_END + "\n")
        close_tags = self.remove_items_in_codeblock(close_tags)
        for close_tag in close_tags:
            if start < close_tag.begin():
                return close_tag

    # Search MarkdownTOC comments in document
    def find_tag_and_insert(self, edit):
        toc_starts = self.get_toc_open_tag()
        for dic in toc_starts:

            toc_start = dic["region"]
            depth     = dic["depth"]
            autolink  = dic["autolink"]

            if 0 < len(toc_start):

                toc_close = self.get_toc_close_tag(toc_start.end())

                if toc_close:
                    toc = self.get_toc(depth, autolink, toc_close.end())
                    tocRegion = sublime.Region(
                        toc_start.end(), toc_close.begin())
                    if toc:
                        self.view.replace(edit, tocRegion, "\n" + toc + "\n")
                        log('refresh TOC content')
                        return True
                    else:
                        self.view.replace(edit, tocRegion, "\n")
                        log('TOC is empty')
                        return False
        log('cannot find TOC tags')
        return False

    # TODO: add "end" parameter
    def get_toc(self, depth, autolink, begin):

        # Search headings in docment
        if depth == 0:
            pattern_hash = "^#+?[^#]"
        else:
            pattern_hash = "^#{1," + str(depth) + "}[^#]"
        headings = self.view.find_all(
            "%s|%s" % (pattern_h1_h2_equal_dash, pattern_hash))

        headings = self.remove_items_in_codeblock(headings)

        if len(headings) < 1:
            return False

        items = []  # [[headingNum,text],...]
        for heading in headings:
            if begin < heading.end():
                lines = self.view.lines(heading)
                if len(lines) == 1:
                    # handle hash headings, ### chapter 1
                    r = sublime.Region(
                        heading.end(), self.view.line(heading).end())
                    heading_text = self.view.substr(r)
                    heading_num = heading.size() - 1
                    items.append([heading_num, heading_text])
                elif len(lines) == 2:
                    # handle - or + headings, Title 1==== section1----
                    heading_text = self.view.substr(lines[0])
                    if heading_text.strip():
                        heading_num = 1 if (
                            self.view.substr(lines[1])[0] == '=') else 2
                        items.append([heading_num, heading_text])

        if len(items) < 1:
            return

        # Shape TOC  ------------------
        items = format(items)

        # Create TOC  ------------------
        toc = ''
        id_texts = []
        for item in items:
            heading_num = item[0] - 1
            heading_text = item[1].rstrip()

            # add indent by heading_num
            for i in range(heading_num):
                toc += '\t'

            # Handling anchors ("Reference-style links")
            matchObj = pattern_anchor.search(heading_text)
            if matchObj:
                only_text = heading_text[0:matchObj.start()]
                only_text = only_text.rstrip()
                id_text = matchObj.group().replace('[','').replace(']','')
                toc += '- [' + only_text + '](#' + id_text + ')\n'
            elif autolink:
                id_text = remove_reserved_chars(heading_text.lower().replace(" ", "-"))
                id_texts.append(id_text)
                n = id_texts.count(id_text)
                if 1 < n:
                    id_text += '-' + str(n-1)
                toc += '- [' + heading_text + '](#' + id_text + ')\n'
            else:
                toc += '- ' + heading_text + '\n'

        return toc

    def get_default_depth(self):
        setting_file = 'MarkdownTOC.sublime-settings'
        settings = sublime.load_settings(setting_file)
        default_depth = settings.get('default_depth')
        if default_depth is None:
            default_depth = 2
            # Save "Settings - Default"
            settings.set('default_depth', default_depth)
            sublime.save_settings(setting_file)
        return default_depth

    def remove_items_in_codeblock(self, items):

        codeblocks = self.view.find_all("^`{3,}[^`]*$")
        codeblockAreas = [] # [[area_begin, area_end], ..]
        i = 0
        while i < len(codeblocks)-1:
            area_begin = codeblocks[i].begin()
            area_end   = codeblocks[i+1].begin()
            if area_begin and area_end:
                codeblockAreas.append([area_begin, area_end])
            i += 2

        items = [h for h in items if is_out_of_areas(h.begin(), codeblockAreas)]
        return items

def remove_reserved_chars(str):
    # Percent-encoding reserved characters
    delete = {
        ord(u"!"): None,
        ord(u"#"): None,
        ord(u"$"): None,
        ord(u"&"): None,
        ord(u"'"): None,
        ord(u"("): None,
        ord(u")"): None,
        ord(u"*"): None,
        ord(u"+"): None,
        ord(u","): None,
        ord(u"/"): None,
        ord(u":"): None,
        ord(u";"): None,
        ord(u"="): None,
        ord(u"?"): None,
        ord(u"@"): None,
        ord(u"["): None,
        ord(u"]"): None,
        ord(u"`"): None
    }
    return str.translate(delete)

def is_out_of_areas(num, areas):
    for area in areas:
        if area[0] < num and num < area[1]:
            return False
    return True

def format(items):
    headings = []
    for item in items:
        headings.append(item[0])
    # ----------

    # set root to 1
    min_heading = min(headings)
    if 1 < min_heading:
        for i, item in enumerate(headings):
            headings[i] -= min_heading - 1
    headings[0] = 1  # first item must be 1

    # minimize "jump width"
    for i, item in enumerate(headings):
        if 0 < i and 1 < item - headings[i - 1]:
            before = headings[i]
            after = headings[i - 1] + 1
            headings[i] = after
            for n in range(i + 1, len(headings)):
                if(headings[n] == before):
                    headings[n] = after
                else:
                    break

    # ----------
    for i, item in enumerate(items):
        item[0] = headings[i]
    return items

def log(arg):
    sublime.status_message(arg)
    print(arg)

# Search and refresh if it's exist

class MarkdowntocUpdate(MarkdowntocInsert):

    def run(self, edit):
        MarkdowntocInsert.find_tag_and_insert(self, edit)

class AutoRunner(sublime_plugin.EventListener):

    def on_pre_save(self, view):
        # limit scope
        root, ext = os.path.splitext(view.file_name())
        ext = ext.lower()
        if ext in [".md", ".markdown", ".mdown", ".mdwn", ".mkdn", ".mkd", ".mark"]:
            view.run_command('markdowntoc_update')
<|endoftext|>


<commit_before>import scipy._lib.uarray as ua
from . import _pocketfft

class _ScipyBackend:
    """The default backend for fft calculations

    Notes
    -----
    We use the domain ``numpy.scipy`` rather than ``scipy`` because in the
    future, ``uarray`` will treat the domain as a hierarchy. This means the user
    can install a single backend for ``numpy`` and have it implement
    ``numpy.scipy.fft`` as well.
    """
    __ua_domain__ = "numpy.scipy.fft"

    @staticmethod
    def __ua_function__(method, args, kwargs):
        fn = getattr(_pocketfft, method.__name__, None)

        if fn is None:
            return NotImplemented
        return fn(*args, **kwargs)

_named_backends = {
    'scipy': _ScipyBackend,
}

def _backend_from_arg(backend):
    """Maps strings to known backends and validates the backend"""

    if isinstance(backend, str):
        try:
            backend = _named_backends[backend]
        except KeyError as e:
            raise ValueError('Unknown backend {}'.format(backend)) from e

    if backend.__ua_domain__ != 'numpy.scipy.fft':
        raise ValueError('Backend does not implement "numpy.scipy.fft"')

    return backend

def set_global_backend(backend):
    """Sets the global fft backend

    The global backend has higher priority than registered backends, but lower
    priority than context-specific backends set with `set_backend`.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.

    Raises
    ------
    ValueError: If the backend does not implement ``numpy.scipy.fft``.

    Notes
    -----
    This will overwrite the previously set global backend, which, by default, is
    the SciPy implementation.

    Examples
    --------
    We can set the global fft backend:

    >>> from scipy.fft import fft, set_global_backend
    >>> set_global_backend("scipy")  # Sets global backend. "scipy" is the default backend.
    >>> fft([1])  # Calls the global backend
    array([1.+0.j])
    """
    backend = _backend_from_arg(backend)
    ua.set_global_backend(backend)

def register_backend(backend):
    """
    Register a backend for permanent use.

    Registered backends have the lowest priority and will be tried after the
    global backend.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.

    Raises
    ------
    ValueError: If the backend does not implement ``numpy.scipy.fft``.

    Examples
    --------
    We can register a new fft backend:

    >>> from scipy.fft import fft, register_backend, set_global_backend
    >>> class NoopBackend:  # Define an invalid Backend
    ...     __ua_domain__ = "numpy.scipy.fft"
    ...     def __ua_function__(self, func, args, kwargs):
    ...          return NotImplemented
    >>> set_global_backend(NoopBackend())  # Set the invalid backend as global
    >>> register_backend("scipy")  # Register a new backend
    >>> fft([1])  # The registered backend is called because the global backend returns `NotImplemented`
    array([1.+0.j])
    >>> set_global_backend("scipy")  # Restore global backend to default

    """
    backend = _backend_from_arg(backend)
    ua.register_backend(backend)

def set_backend(backend, coerce=False, only=False):
    """Context manager to set the backend within a fixed scope.

    Upon entering the ``with`` statement, the given backend will be added to
    the list of available backends with the highest priority. Upon exit, the
    backend is reset to the state before entering the scope.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.
    coerce : bool, optional
        Whether to allow expensive conversions for the ``x`` parameter. e.g.,
        copying a NumPy array to the GPU for a CuPy backend. Implies ``only``.
    only : bool, optional
        If only is ``True`` and this backend returns ``NotImplemented``, then a
        BackendNotImplemented error will be raised immediately. Ignoring any
        lower priority backends.

    Examples
    --------
    >>> import scipy.fft as fft
    >>> with fft.set_backend('scipy', only=True):
    ...     fft.fft([1])  # Always calls the scipy implementation
    array([1.+0.j])
    """
    backend = _backend_from_arg(backend)
    return ua.set_backend(backend, coerce=coerce, only=only)

def skip_backend(backend):
    """Context manager to skip a backend within a fixed scope.

    Within the context of a ``with`` statement, the given backend will not be
    called. This covers backends registered both locally and globally. Upon
    exit, the backend will again be considered.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to skip.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.

    Examples
    --------
    >>> import scipy.fft as fft
    >>> fft.fft([1])  # Calls default SciPy backend
    array([1.+0.j])
    >>> with fft.skip_backend('scipy'):  # We explicitly skip the SciPy backend
    ...     fft.fft([1])                 # leaving no implementation available
    Traceback (most recent call last):
        ...
    BackendNotImplementedError: No selected backends had an implementation ...
    """
    backend = _backend_from_arg(backend)
    return ua.skip_backend(backend)

set_global_backend('scipy')
<commit_msg>ENH: Set fft backend with try_last=True (#14359)<commit_after>import scipy._lib.uarray as ua
from . import _pocketfft

class _ScipyBackend:
    """The default backend for fft calculations

    Notes
    -----
    We use the domain ``numpy.scipy`` rather than ``scipy`` because ``uarray``
    treats the domain as a hierarchy. This means the user can install a single
    backend for ``numpy`` and have it implement ``numpy.scipy.fft`` as well.
    """
    __ua_domain__ = "numpy.scipy.fft"

    @staticmethod
    def __ua_function__(method, args, kwargs):
        fn = getattr(_pocketfft, method.__name__, None)

        if fn is None:
            return NotImplemented
        return fn(*args, **kwargs)

_named_backends = {
    'scipy': _ScipyBackend,
}

def _backend_from_arg(backend):
    """Maps strings to known backends and validates the backend"""

    if isinstance(backend, str):
        try:
            backend = _named_backends[backend]
        except KeyError as e:
            raise ValueError('Unknown backend {}'.format(backend)) from e

    if backend.__ua_domain__ != 'numpy.scipy.fft':
        raise ValueError('Backend does not implement "numpy.scipy.fft"')

    return backend

def set_global_backend(backend, coerce=False, only=False, try_last=False):
    """Sets the global fft backend

    This utility method replaces the default backend for permanent use. It
    will be tried in the list of backends automatically, unless the
    ``only`` flag is set on a backend. This will be the first tried
    backend outside the :obj:`set_backend` context manager.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.
    coerce : bool
        Whether to coerce input types when trying this backend.
    only : bool
        If ``True``, no more backends will be tried if this fails.
        Implied by ``coerce=True``.
    try_last : bool
        If ``True``, the global backend is tried after registered backends.

    Raises
    ------
    ValueError: If the backend does not implement ``numpy.scipy.fft``.

    Notes
    -----
    This will overwrite the previously set global backend, which, by default, is
    the SciPy implementation.

    Examples
    --------
    We can set the global fft backend:

    >>> from scipy.fft import fft, set_global_backend
    >>> set_global_backend("scipy")  # Sets global backend. "scipy" is the default backend.
    >>> fft([1])  # Calls the global backend
    array([1.+0.j])
    """
    backend = _backend_from_arg(backend)
    ua.set_global_backend(backend, coerce=coerce, only=only, try_last=try_last)

def register_backend(backend):
    """
    Register a backend for permanent use.

    Registered backends have the lowest priority and will be tried after the
    global backend.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.

    Raises
    ------
    ValueError: If the backend does not implement ``numpy.scipy.fft``.

    Examples
    --------
    We can register a new fft backend:

    >>> from scipy.fft import fft, register_backend, set_global_backend
    >>> class NoopBackend:  # Define an invalid Backend
    ...     __ua_domain__ = "numpy.scipy.fft"
    ...     def __ua_function__(self, func, args, kwargs):
    ...          return NotImplemented
    >>> set_global_backend(NoopBackend())  # Set the invalid backend as global
    >>> register_backend("scipy")  # Register a new backend
    >>> fft([1])  # The registered backend is called because the global backend returns `NotImplemented`
    array([1.+0.j])
    >>> set_global_backend("scipy")  # Restore global backend to default

    """
    backend = _backend_from_arg(backend)
    ua.register_backend(backend)

def set_backend(backend, coerce=False, only=False):
    """Context manager to set the backend within a fixed scope.

    Upon entering the ``with`` statement, the given backend will be added to
    the list of available backends with the highest priority. Upon exit, the
    backend is reset to the state before entering the scope.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to use.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.
    coerce : bool, optional
        Whether to allow expensive conversions for the ``x`` parameter. e.g.,
        copying a NumPy array to the GPU for a CuPy backend. Implies ``only``.
    only : bool, optional
        If only is ``True`` and this backend returns ``NotImplemented``, then a
        BackendNotImplemented error will be raised immediately. Ignoring any
        lower priority backends.

    Examples
    --------
    >>> import scipy.fft as fft
    >>> with fft.set_backend('scipy', only=True):
    ...     fft.fft([1])  # Always calls the scipy implementation
    array([1.+0.j])
    """
    backend = _backend_from_arg(backend)
    return ua.set_backend(backend, coerce=coerce, only=only)

def skip_backend(backend):
    """Context manager to skip a backend within a fixed scope.

    Within the context of a ``with`` statement, the given backend will not be
    called. This covers backends registered both locally and globally. Upon
    exit, the backend will again be considered.

    Parameters
    ----------
    backend : {object, 'scipy'}
        The backend to skip.
        Can either be a ``str`` containing the name of a known backend
        {'scipy'} or an object that implements the uarray protocol.

    Examples
    --------
    >>> import scipy.fft as fft
    >>> fft.fft([1])  # Calls default SciPy backend
    array([1.+0.j])
    >>> with fft.skip_backend('scipy'):  # We explicitly skip the SciPy backend
    ...     fft.fft([1])                 # leaving no implementation available
    Traceback (most recent call last):
        ...
    BackendNotImplementedError: No selected backends had an implementation ...
    """
    backend = _backend_from_arg(backend)
    return ua.skip_backend(backend)

set_global_backend('scipy', try_last=True)
<|endoftext|>


<commit_before>import scipy
import scipy.sparse as sps

from sfepy.base.base import output, get_default, Struct
from sfepy.solvers.solvers import LinearSolver

def try_imports(imports, fail_msg=None):
    for imp in imports:
        try:
            exec imp
            break
        except:
            pass
        else:
            if fail_msg is not None:
                raise ValueError(fail_msg)
    return locals()

class ScipyDirect(LinearSolver):
    name = 'ls.scipy_direct'

    def process_conf(conf):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_1100 = {
                'name' : 'dls1100',
                'kind' : 'ls.scipy_direct',

                'method' : 'superlu',
                'presolve' : False,
                'warn' : True,
            }
        """
        get = conf.get_default_attr

        method = get('method', 'auto')
        presolve = get('presolve', False)
        warn = get('warn', True)

        common = LinearSolver.process_conf(conf)
        return Struct(**locals()) + common
    process_conf = staticmethod(process_conf)

    def __init__(self, conf, **kwargs):
        LinearSolver.__init__(self, conf, **kwargs)

        um = self.sls = None

        aux = try_imports(['import scipy.linsolve as sls',
                           'import scipy.splinalg.dsolve as sls',
                           'import scipy.sparse.linalg.dsolve as sls'],
                          'cannot import scipy sparse direct solvers!')
        self.sls = aux['sls']
        aux = try_imports(['import scipy.linsolve.umfpack as um',
                           'import scipy.splinalg.dsolve.umfpack as um',
                           'import scipy.sparse.linalg.dsolve.umfpack as um',
                           'import scikits.umfpack as um'])
        if 'um' in aux:
            um = aux['um']

        if um is not None:
            is_umfpack = hasattr(um, 'UMFPACK_OK')
        else:
            is_umfpack = False

        method = self.conf.method
        if method == 'superlu':
            self.sls.use_solver(useUmfpack=False)
        elif method == 'umfpack':
            if not is_umfpack and self.conf.warn:
                output('umfpack not available, using superlu!')
        elif method != 'auto':
            raise ValueError('uknown solution method! (%s)' % method)

        if method != 'superlu' and is_umfpack:
            self.sls.use_solver(useUmfpack=True,
                                assumeSortedIndices=True)

        self.solve = None
        if self._presolve() and hasattr(self, 'mtx'):
            if self.mtx is not None:
                self.solve = self.sls.factorized(self.mtx)

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        if self.solve is not None:
            # Matrix is already prefactorized.
            return self.solve(rhs)
        else:
            return self.sls.spsolve(mtx, rhs)

    def _presolve(self):
        if hasattr(self, 'presolve'):
            return self.presolve
        else:
            return self.conf.presolve

class Umfpack(ScipyDirect):
    """This class stays for compatability with old input files. Use ScipyDirect
    isntead."""
    name = 'ls.umfpack'

    def __init__(self, conf, **kwargs):
        conf.method = 'umfpack'
        ScipyDirect.__init__(self, conf, **kwargs)

##
# c: 22.02.2008
class ScipyIterative( LinearSolver ):
    """
    Interface to scipy iterative solvers.

    Notes
    -----
    The `eps_r` tolerance is both absolute and relative - the solvers
    stop when either the relative or the absolute residual is below it.
    """
    name = 'ls.scipy_iterative'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_110 = {
                'name' : 'ls110',
                'kind' : 'ls.scipy_iterative',

                'method' : 'cg',
                'i_max' : 1000,
                'eps_r' : 1e-12,
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'cg' )
        i_max = get( 'i_max', 100 )
        eps_r = get('eps_r', 1e-8)

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    ##
    # c: 22.02.2008, r: 23.06.2008
    def __init__( self, conf, **kwargs ):
        if scipy.version.version < '0.7.0.dev3861':
            import scipy.linalg as la
        else:
            if scipy.version.version < '0.7.0.dev3998':
                import scipy.splinalg.isolve as la
            else:
                import scipy.sparse.linalg.isolve as la

        LinearSolver.__init__(self, conf,
                              eps_a=conf.eps_r, eps_r=conf.eps_r, **kwargs)

        try:
            solver = getattr( la, self.conf.method )
        except AttributeError:
            output( 'scipy solver %s does not exist!' % self.conf.method )
            output( 'using cg instead' )
            solver = la.cg
        self.solver = solver

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_a = get_default(eps_a, self.eps_a)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        sol, info = self.solver(mtx, rhs, x0=x0, tol=max(eps_a, eps_r),
                                maxiter=conf.i_max)

        return sol

##
# c: 02.05.2008, r: 02.05.2008
class PyAMGSolver( LinearSolver ):
    """
    Interface to PyAMG solvers.

    Notes
    -----
    Uses relative convergence tolerance, i.e. eps_r is scaled by `||b||`.
    """
    name = 'ls.pyamg'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_102 = {
                'name' : 'ls102',
                'kind' : 'ls.pyamg',

                'method' : 'smoothed_aggregation_solver',
                'accel' : 'cg'
                'eps_r' : 1e-12,
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'smoothed_aggregation_solver' )
        accel = get( 'accel', None )
        eps_r = get('eps_r', 1e-8)

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    ##
    # c: 02.05.2008, r: 02.05.2008
    def __init__( self, conf, **kwargs ):
        try:
            import pyamg
        except ImportError:
            msg =  'cannot import pyamg!'
            raise ImportError( msg )

        LinearSolver.__init__(self, conf, eps_r=conf.eps_r, mg=None, **kwargs)

        try:
            solver = getattr( pyamg, self.conf.method )
        except AttributeError:
            output( 'pyamg.%s does not exist!' % self.conf.method )
            output( 'using pyamg.smoothed_aggregation_solver instead' )
            solver = pyamg.smoothed_aggregation_solver
        self.solver = solver

        if hasattr( self, 'mtx' ):
            if self.mtx is not None:
                self.mg = self.solver( self.mtx )

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        if (self.mg is None) or (mtx is not self.mtx):
            self.mg = self.solver(mtx)
            self.mtx = mtx

        sol = self.mg.solve(rhs, x0=x0, accel=conf.accel, tol=eps_r)

        return sol

class PETScKrylovSolver( LinearSolver ):
    """PETSc Krylov subspace solver.

    The solver and preconditioner types are set upon the solver object
    creation. Tolerances can be overriden when called by passing a conf
    object.

    Convergence is reached when 'rnorm < max(eps_r * rnorm_0, eps_a)', where
    rnorm is the norm of rezidual, i.e. ||Ax - b||.
    """
    name = 'ls.petsc'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_120 = {
                'name' : 'ls120',
                'kind' : 'ls.petsc',

                'method' : 'cg', # ksp_type
                'precond' : 'icc', # pc_type
                'eps_a' : 1e-12, # abstol
                'eps_r' : 1e-12, # rtol
                'i_max' : 1000, # maxits
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'cg' )
        precond = get( 'precond', 'icc' )
        eps_a = get( 'eps_a', 1e-8 )
        eps_r = get( 'eps_r', 1e-8 )
        i_max = get( 'i_max', 100 )

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    def __init__( self, conf, **kwargs ):
        try:
            import petsc4py
            petsc4py.init([])
            from petsc4py import PETSc
        except ImportError:
            msg = 'cannot import petsc4py!'
            raise ImportError( msg )

        LinearSolver.__init__(self, conf, eps_a=conf.eps_a, eps_r=conf.eps_r,
                              petsc=PETSc, pmtx=None, **kwargs)

        ksp = PETSc.KSP().create()

        ksp.setType( self.conf.method )
        ksp.getPC().setType( self.conf.precond )

        if hasattr( self, 'mtx' ):
            if self.mtx is not None:
                self.pmtx, self.sol, self.rhs = self.set_matrix( self.mtx )
                ksp.setOperators( self.pmtx ) # set the matrix
                ksp.setFromOptions()

        self.ksp = ksp

    def set_matrix( self, mtx ):
        mtx = sps.csr_matrix(mtx)

        pmtx = self.petsc.Mat().createAIJ( mtx.shape,
                                           csr = (mtx.indptr,
                                                  mtx.indices,
                                                  mtx.data) )
        sol, rhs = pmtx.getVecs()
        return pmtx, sol, rhs

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_a = get_default(eps_a, self.eps_a)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        ksp = self.ksp
        if (self.pmtx is None) or (mtx is not self.mtx):
            self.pmtx, self.sol, self.rhs = self.set_matrix(mtx)
            self.ksp.setOperators(self.pmtx)
            self.ksp.setFromOptions() # PETSc.Options() not used yet...
            self.mtx = mtx

        ksp.setTolerances(atol=eps_a, rtol=eps_r, max_it=conf.i_max)

        # Set PETSc rhs, solve, get solution from PETSc solution.
        if x0 is not None:
            self.sol[...] = x0
        self.rhs[...] = rhs
        ksp.solve(self.rhs, self.sol)
        sol = self.sol[...].copy()
        output('KSP convergence: %s' % ksp.reason)

        return sol
<commit_msg>improve convergence reporting of ScipyIterative, PETScKrylovSolver<commit_after>import numpy as nm
import scipy
import scipy.sparse as sps

from sfepy.base.base import output, get_default, Struct
from sfepy.solvers.solvers import LinearSolver

def try_imports(imports, fail_msg=None):
    for imp in imports:
        try:
            exec imp
            break
        except:
            pass
        else:
            if fail_msg is not None:
                raise ValueError(fail_msg)
    return locals()

class ScipyDirect(LinearSolver):
    name = 'ls.scipy_direct'

    def process_conf(conf):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_1100 = {
                'name' : 'dls1100',
                'kind' : 'ls.scipy_direct',

                'method' : 'superlu',
                'presolve' : False,
                'warn' : True,
            }
        """
        get = conf.get_default_attr

        method = get('method', 'auto')
        presolve = get('presolve', False)
        warn = get('warn', True)

        common = LinearSolver.process_conf(conf)
        return Struct(**locals()) + common
    process_conf = staticmethod(process_conf)

    def __init__(self, conf, **kwargs):
        LinearSolver.__init__(self, conf, **kwargs)

        um = self.sls = None

        aux = try_imports(['import scipy.linsolve as sls',
                           'import scipy.splinalg.dsolve as sls',
                           'import scipy.sparse.linalg.dsolve as sls'],
                          'cannot import scipy sparse direct solvers!')
        self.sls = aux['sls']
        aux = try_imports(['import scipy.linsolve.umfpack as um',
                           'import scipy.splinalg.dsolve.umfpack as um',
                           'import scipy.sparse.linalg.dsolve.umfpack as um',
                           'import scikits.umfpack as um'])
        if 'um' in aux:
            um = aux['um']

        if um is not None:
            is_umfpack = hasattr(um, 'UMFPACK_OK')
        else:
            is_umfpack = False

        method = self.conf.method
        if method == 'superlu':
            self.sls.use_solver(useUmfpack=False)
        elif method == 'umfpack':
            if not is_umfpack and self.conf.warn:
                output('umfpack not available, using superlu!')
        elif method != 'auto':
            raise ValueError('uknown solution method! (%s)' % method)

        if method != 'superlu' and is_umfpack:
            self.sls.use_solver(useUmfpack=True,
                                assumeSortedIndices=True)

        self.solve = None
        if self._presolve() and hasattr(self, 'mtx'):
            if self.mtx is not None:
                self.solve = self.sls.factorized(self.mtx)

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        if self.solve is not None:
            # Matrix is already prefactorized.
            return self.solve(rhs)
        else:
            return self.sls.spsolve(mtx, rhs)

    def _presolve(self):
        if hasattr(self, 'presolve'):
            return self.presolve
        else:
            return self.conf.presolve

class Umfpack(ScipyDirect):
    """This class stays for compatability with old input files. Use ScipyDirect
    isntead."""
    name = 'ls.umfpack'

    def __init__(self, conf, **kwargs):
        conf.method = 'umfpack'
        ScipyDirect.__init__(self, conf, **kwargs)

##
# c: 22.02.2008
class ScipyIterative( LinearSolver ):
    """
    Interface to scipy iterative solvers.

    Notes
    -----
    The `eps_r` tolerance is both absolute and relative - the solvers
    stop when either the relative or the absolute residual is below it.
    """
    name = 'ls.scipy_iterative'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_110 = {
                'name' : 'ls110',
                'kind' : 'ls.scipy_iterative',

                'method' : 'cg',
                'i_max' : 1000,
                'eps_r' : 1e-12,
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'cg' )
        i_max = get( 'i_max', 100 )
        eps_r = get('eps_r', 1e-8)

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    ##
    # c: 22.02.2008, r: 23.06.2008
    def __init__( self, conf, **kwargs ):
        if scipy.version.version < '0.7.0.dev3861':
            import scipy.linalg as la
        else:
            if scipy.version.version < '0.7.0.dev3998':
                import scipy.splinalg.isolve as la
            else:
                import scipy.sparse.linalg.isolve as la

        LinearSolver.__init__(self, conf,
                              eps_a=conf.eps_r, eps_r=conf.eps_r, **kwargs)

        try:
            solver = getattr( la, self.conf.method )
        except AttributeError:
            output( 'scipy solver %s does not exist!' % self.conf.method )
            output( 'using cg instead' )
            solver = la.cg
        self.solver = solver
        self.converged_reasons = {
            0 : 'successful exit',
            1 : 'number of iterations',
            -1 : 'illegal input or breakdown',
        }

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_a = get_default(eps_a, self.eps_a)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        sol, info = self.solver(mtx, rhs, x0=x0, tol=max(eps_a, eps_r),
                                maxiter=conf.i_max)
        output('%s convergence: %s (%s)'
               % (self.conf.method,
                  info, self.converged_reasons[nm.sign(info)]))

        return sol

##
# c: 02.05.2008, r: 02.05.2008
class PyAMGSolver( LinearSolver ):
    """
    Interface to PyAMG solvers.

    Notes
    -----
    Uses relative convergence tolerance, i.e. eps_r is scaled by `||b||`.
    """
    name = 'ls.pyamg'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_102 = {
                'name' : 'ls102',
                'kind' : 'ls.pyamg',

                'method' : 'smoothed_aggregation_solver',
                'accel' : 'cg'
                'eps_r' : 1e-12,
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'smoothed_aggregation_solver' )
        accel = get( 'accel', None )
        eps_r = get('eps_r', 1e-8)

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    ##
    # c: 02.05.2008, r: 02.05.2008
    def __init__( self, conf, **kwargs ):
        try:
            import pyamg
        except ImportError:
            msg =  'cannot import pyamg!'
            raise ImportError( msg )

        LinearSolver.__init__(self, conf, eps_r=conf.eps_r, mg=None, **kwargs)

        try:
            solver = getattr( pyamg, self.conf.method )
        except AttributeError:
            output( 'pyamg.%s does not exist!' % self.conf.method )
            output( 'using pyamg.smoothed_aggregation_solver instead' )
            solver = pyamg.smoothed_aggregation_solver
        self.solver = solver

        if hasattr( self, 'mtx' ):
            if self.mtx is not None:
                self.mg = self.solver( self.mtx )

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        if (self.mg is None) or (mtx is not self.mtx):
            self.mg = self.solver(mtx)
            self.mtx = mtx

        sol = self.mg.solve(rhs, x0=x0, accel=conf.accel, tol=eps_r)

        return sol

class PETScKrylovSolver( LinearSolver ):
    """PETSc Krylov subspace solver.

    The solver and preconditioner types are set upon the solver object
    creation. Tolerances can be overriden when called by passing a conf
    object.

    Convergence is reached when 'rnorm < max(eps_r * rnorm_0, eps_a)', where
    rnorm is the norm of rezidual, i.e. ||Ax - b||.
    """
    name = 'ls.petsc'

    def process_conf( conf ):
        """
        Missing items are set to default values.

        Example configuration, all items::

            solver_120 = {
                'name' : 'ls120',
                'kind' : 'ls.petsc',

                'method' : 'cg', # ksp_type
                'precond' : 'icc', # pc_type
                'eps_a' : 1e-12, # abstol
                'eps_r' : 1e-12, # rtol
                'i_max' : 1000, # maxits
            }
        """
        get = conf.get_default_attr

        method = get( 'method', 'cg' )
        precond = get( 'precond', 'icc' )
        eps_a = get( 'eps_a', 1e-8 )
        eps_r = get( 'eps_r', 1e-8 )
        i_max = get( 'i_max', 100 )

        common = LinearSolver.process_conf( conf )
        return Struct( **locals() ) + common
    process_conf = staticmethod( process_conf )

    def __init__( self, conf, **kwargs ):
        try:
            import petsc4py
            petsc4py.init([])
            from petsc4py import PETSc
        except ImportError:
            msg = 'cannot import petsc4py!'
            raise ImportError( msg )

        LinearSolver.__init__(self, conf, eps_a=conf.eps_a, eps_r=conf.eps_r,
                              petsc=PETSc, pmtx=None, **kwargs)

        ksp = PETSc.KSP().create()

        ksp.setType( self.conf.method )
        ksp.getPC().setType( self.conf.precond )

        if hasattr( self, 'mtx' ):
            if self.mtx is not None:
                self.pmtx, self.sol, self.rhs = self.set_matrix( self.mtx )
                ksp.setOperators( self.pmtx ) # set the matrix
                ksp.setFromOptions()

        self.ksp = ksp

        self.converged_reasons = {}
        for key, val in ksp.ConvergedReason.__dict__.iteritems():
            if isinstance(val, int):
                self.converged_reasons[val] = key

    def set_matrix( self, mtx ):
        mtx = sps.csr_matrix(mtx)

        pmtx = self.petsc.Mat().createAIJ( mtx.shape,
                                           csr = (mtx.indptr,
                                                  mtx.indices,
                                                  mtx.data) )
        sol, rhs = pmtx.getVecs()
        return pmtx, sol, rhs

    def __call__(self, rhs, x0=None, conf=None, eps_a=None, eps_r=None,
                 mtx=None, status=None):
        conf = get_default(conf, self.conf)
        eps_a = get_default(eps_a, self.eps_a)
        eps_r = get_default(eps_r, self.eps_r)
        mtx = get_default(mtx, self.mtx)
        status = get_default(status, self.status)

        ksp = self.ksp
        if (self.pmtx is None) or (mtx is not self.mtx):
            self.pmtx, self.sol, self.rhs = self.set_matrix(mtx)
            self.ksp.setOperators(self.pmtx)
            self.ksp.setFromOptions() # PETSc.Options() not used yet...
            self.mtx = mtx

        ksp.setTolerances(atol=eps_a, rtol=eps_r, max_it=conf.i_max)

        # Set PETSc rhs, solve, get solution from PETSc solution.
        if x0 is not None:
            self.sol[...] = x0
        self.rhs[...] = rhs
        ksp.solve(self.rhs, self.sol)
        sol = self.sol[...].copy()
        output('%s(%s) convergence: %s (%s)'
               % (self.conf.method, self.conf.precond,
                  ksp.reason, self.converged_reasons[ksp.reason]))

        return sol
<|endoftext|>


<commit_before>"""
Measurement filters
"""

from atom.api import Atom, Int, Float, List, Str, Dict, Bool, Enum, Coerced, Typed, observe
import enaml
from enaml.qt.qt_application import QtApplication

from DictManager import DictManager
import json

class MeasFilter(Atom):
    label = Str()
    enabled = Bool(True)
    plotScope = Bool(False).tag(desc='Whether to show the raw data scope.')
    plotMode = Enum('amp/phase', 'real/imag', 'quad').tag(desc='Filtered data scope mode.')
    saved = Bool(True).tag(desc='Whether the filtered values should be saved to file.')
    dataSource = Str().tag(desc="Where the measurement data is pushed from.")

    def json_encode(self, matlabCompatible=False):
        jsonDict = self.__getstate__()
        if matlabCompatible:
            jsonDict['filterType'] = self.__class__.__name__
            jsonDict.pop('enabled', None)
            jsonDict.pop('label', None)
        else:
            jsonDict['x__class__'] = self.__class__.__name__
            jsonDict['x__module__'] = self.__class__.__module__
        return jsonDict

class RawStream(MeasFilter):
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')
    channel = Str().tag(desc="The channel on the digitizer to pull data from.")

class DigitalDemod(MeasFilter):
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')
    IFfreq = Float(10e6).tag(desc='The I.F. frequency for digital demodulation.')
    bandwidth = Float(5e6).tag(desc='Low-pass filter bandwidth')
    samplingRate = Float(250e6).tag(desc='The sampling rate of the digitizer.')
    phase = Float(0.0).tag(desc='Phase rotation to apply in rad.')
    decimFactor1 = Int(1).tag(desc="First stage polyphase decimation (before multiplication with reference).")
    decimFactor2 = Int(1).tag(desc="Second stage polyphase decimation (before IIR filter).")
    decimFactor3 = Int(1).tag(desc="Third stage polyphase decimation (after IIR filter).")

class KernelIntegration(MeasFilter):
    kernel = Str('').tag(desc="Integration kernel vector.")
    bias = Float(0.0).tag(desc="Bias after integration.")
    simpleKernel = Bool(True)
    boxCarStart = Int(1)
    boxCarStop = Int(1)
    IFfreq = Float(10e6)
    samplingRate = Float(250e6)

    def json_encode(self, matlabCompatible=False):
        jsonDict = super(KernelIntegration, self).json_encode(matlabCompatible)
        if matlabCompatible:
            # re-encode kernel in base64
            jsonDict.pop('kernel')

            import numpy as np
            import base64
            try:
                if self.simpleKernel:
                    kernel = np.hstack((np.zeros(self.boxCarStart, dtype=np.complex), np.ones(self.boxCarStop-self.boxCarStart)))
                    kernel *= np.exp(1j*2*np.pi*self.IFfreq*np.arange(self.boxCarStop))
                    kernel = base64.b64encode(kernel)
                else:
                    kernel = base64.b64encode(eval(self.kernel))
            except:
                kernel = []
            jsonDict['kernel'] = kernel
        return jsonDict

class Correlator(MeasFilter):
    filters = List()

    def __init__(self, **kwargs):
        super(Correlator, self).__init__(**kwargs)
        if not self.filters:
            self.filters = [None, None]

    def json_encode(self, matlabCompatible=False):
        jsonDict = super(Correlator, self).json_encode(matlabCompatible)
        #For correlation filters return the filter list as a list of filter names
        filterList = jsonDict.pop('filters')
        jsonDict['filters'] = [item.label for item in filterList] if filterList else []
        jsonDict.pop('channel')
        jsonDict.pop('plotScope')
        return jsonDict

class StateComparator(MeasFilter):
    threshold = Float(0.0)
    integrationTime = Int(-1).tag(desc='Comparator integration time in decimated samples, use -1 for the entire record')

class StreamSelector(MeasFilter):
    stream = Str()
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')

class MeasFilterLibrary(Atom):
    # filterDict = Dict(Str, MeasFilter)
    filterDict = Coerced(dict)
    libFile = Str().tag(transient=True)
    filterManager = Typed(DictManager)

    def __init__(self, **kwargs):
        super(MeasFilterLibrary, self).__init__(**kwargs)
        self.load_from_library()
        self.filterManager = DictManager(itemDict=self.filterDict, possibleItems=measFilterList)

    #Overload [] to allow direct pulling of measurement filter info
    def __getitem__(self, filterName):
        return self.filterDict[filterName]

    def write_to_file(self):
        #Move import here to avoid circular import
        import JSONHelpers
        if self.libFile:
            with open(self.libFile,'w') as FID:
                json.dump(self, FID, cls=JSONHelpers.LibraryEncoder, indent=2, sort_keys=True)

    def load_from_library(self):
        import JSONHelpers
        if self.libFile:
            try:
                with open(self.libFile, 'r') as FID:
                    tmpLib = json.load(FID, cls=JSONHelpers.LibraryDecoder)
                    if isinstance(tmpLib, MeasFilterLibrary):
                        #Update correlator filter lists to filter objects
                        for filt in tmpLib.filterDict.values():
                            if isinstance(filt, Correlator):
                                filterList = []
                                for f in filt.filters:
                                    filterList.append(tmpLib.filterDict[f])
                                filt.filters = filterList
                        self.filterDict.update(tmpLib.filterDict)
            except IOError:
                print("No measurement library found.")

    def json_encode(self, matlabCompatible=False):
        if matlabCompatible:
            return {label:filt for label,filt in self.filterDict.items() if filt.enabled}
        else:
            return {"filterDict":{label:filt for label,filt in self.filterDict.items()}}

measFilterList = [RawStream, DigitalDemod, KernelIntegration, Correlator, StateComparator, StreamSelector]

if __name__ == "__main__":

    #Work around annoying problem with multiple class definitions
    from MeasFilters import DigitalHomodyne, Correlator, MeasFilterLibrary

    testFilter1 = DigitalHomodyne(label='M1', boxCarStart=100, boxCarStop=500, IFfreq=10e6, samplingRate=250e6, channel=1)
    testFilter2 = DigitalHomodyne(label='M2', boxCarStart=150, boxCarStop=600, IFfreq=39.2e6, samplingRate=250e6, channel=2)
    testFilter3 = Correlator(label='M12')
    filterDict = {'M1':testFilter1, 'M2':testFilter2, 'M12':testFilter3}

    testLib = MeasFilterLibrary(libFile='MeasFilterLibrary.json', filterDict=filterDict)

    with enaml.imports():
        from MeasFiltersViews import MeasFilterManagerWindow

    app = QtApplication()
    view = MeasFilterManagerWindow(filterLib=testLib)
    view.show()

    app.start()
<commit_msg>Use samplingRate in simple kernel calculation.<commit_after>"""
Measurement filters
"""

from atom.api import Atom, Int, Float, List, Str, Dict, Bool, Enum, Coerced, Typed, observe
import enaml
from enaml.qt.qt_application import QtApplication

from DictManager import DictManager
import json

class MeasFilter(Atom):
    label = Str()
    enabled = Bool(True)
    plotScope = Bool(False).tag(desc='Whether to show the raw data scope.')
    plotMode = Enum('amp/phase', 'real/imag', 'quad').tag(desc='Filtered data scope mode.')
    saved = Bool(True).tag(desc='Whether the filtered values should be saved to file.')
    dataSource = Str().tag(desc="Where the measurement data is pushed from.")

    def json_encode(self, matlabCompatible=False):
        jsonDict = self.__getstate__()
        if matlabCompatible:
            jsonDict['filterType'] = self.__class__.__name__
            jsonDict.pop('enabled', None)
            jsonDict.pop('label', None)
        else:
            jsonDict['x__class__'] = self.__class__.__name__
            jsonDict['x__module__'] = self.__class__.__module__
        return jsonDict

class RawStream(MeasFilter):
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')
    channel = Str().tag(desc="The channel on the digitizer to pull data from.")

class DigitalDemod(MeasFilter):
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')
    IFfreq = Float(10e6).tag(desc='The I.F. frequency for digital demodulation.')
    bandwidth = Float(5e6).tag(desc='Low-pass filter bandwidth')
    samplingRate = Float(250e6).tag(desc='The sampling rate of the digitizer.')
    phase = Float(0.0).tag(desc='Phase rotation to apply in rad.')
    decimFactor1 = Int(1).tag(desc="First stage polyphase decimation (before multiplication with reference).")
    decimFactor2 = Int(1).tag(desc="Second stage polyphase decimation (before IIR filter).")
    decimFactor3 = Int(1).tag(desc="Third stage polyphase decimation (after IIR filter).")

class KernelIntegration(MeasFilter):
    kernel = Str('').tag(desc="Integration kernel vector.")
    bias = Float(0.0).tag(desc="Bias after integration.")
    simpleKernel = Bool(True)
    boxCarStart = Int(1)
    boxCarStop = Int(1)
    IFfreq = Float(10e6)
    samplingRate = Float(250e6)

    def json_encode(self, matlabCompatible=False):
        jsonDict = super(KernelIntegration, self).json_encode(matlabCompatible)
        if matlabCompatible:
            # re-encode kernel in base64
            jsonDict.pop('kernel')

            import numpy as np
            import base64
            try:
                if self.simpleKernel:
                    kernel = np.hstack((np.zeros(self.boxCarStart, dtype=np.complex), np.ones(self.boxCarStop-self.boxCarStart)))
                    kernel *= np.exp(1j*2*np.pi*self.IFfreq*np.arange(self.boxCarStop)/self.samplingRate)
                    kernel = base64.b64encode(kernel)
                else:
                    kernel = base64.b64encode(eval(self.kernel))
            except:
                kernel = []
            jsonDict['kernel'] = kernel
        return jsonDict

class Correlator(MeasFilter):
    filters = List()

    def __init__(self, **kwargs):
        super(Correlator, self).__init__(**kwargs)
        if not self.filters:
            self.filters = [None, None]

    def json_encode(self, matlabCompatible=False):
        jsonDict = super(Correlator, self).json_encode(matlabCompatible)
        #For correlation filters return the filter list as a list of filter names
        filterList = jsonDict.pop('filters')
        jsonDict['filters'] = [item.label for item in filterList] if filterList else []
        jsonDict.pop('channel')
        jsonDict.pop('plotScope')
        return jsonDict

class StateComparator(MeasFilter):
    threshold = Float(0.0)
    integrationTime = Int(-1).tag(desc='Comparator integration time in decimated samples, use -1 for the entire record')

class StreamSelector(MeasFilter):
    stream = Str()
    saveRecords = Bool(False).tag(desc='Whether to save the single-shot records to file.')
    recordsFilePath = Str('').tag(desc='Path to file where records will be optionally saved.')

class MeasFilterLibrary(Atom):
    # filterDict = Dict(Str, MeasFilter)
    filterDict = Coerced(dict)
    libFile = Str().tag(transient=True)
    filterManager = Typed(DictManager)

    def __init__(self, **kwargs):
        super(MeasFilterLibrary, self).__init__(**kwargs)
        self.load_from_library()
        self.filterManager = DictManager(itemDict=self.filterDict, possibleItems=measFilterList)

    #Overload [] to allow direct pulling of measurement filter info
    def __getitem__(self, filterName):
        return self.filterDict[filterName]

    def write_to_file(self):
        #Move import here to avoid circular import
        import JSONHelpers
        if self.libFile:
            with open(self.libFile,'w') as FID:
                json.dump(self, FID, cls=JSONHelpers.LibraryEncoder, indent=2, sort_keys=True)

    def load_from_library(self):
        import JSONHelpers
        if self.libFile:
            try:
                with open(self.libFile, 'r') as FID:
                    tmpLib = json.load(FID, cls=JSONHelpers.LibraryDecoder)
                    if isinstance(tmpLib, MeasFilterLibrary):
                        #Update correlator filter lists to filter objects
                        for filt in tmpLib.filterDict.values():
                            if isinstance(filt, Correlator):
                                filterList = []
                                for f in filt.filters:
                                    filterList.append(tmpLib.filterDict[f])
                                filt.filters = filterList
                        self.filterDict.update(tmpLib.filterDict)
            except IOError:
                print("No measurement library found.")

    def json_encode(self, matlabCompatible=False):
        if matlabCompatible:
            return {label:filt for label,filt in self.filterDict.items() if filt.enabled}
        else:
            return {"filterDict":{label:filt for label,filt in self.filterDict.items()}}

measFilterList = [RawStream, DigitalDemod, KernelIntegration, Correlator, StateComparator, StreamSelector]

if __name__ == "__main__":

    #Work around annoying problem with multiple class definitions
    from MeasFilters import DigitalHomodyne, Correlator, MeasFilterLibrary

    testFilter1 = DigitalHomodyne(label='M1', boxCarStart=100, boxCarStop=500, IFfreq=10e6, samplingRate=250e6, channel=1)
    testFilter2 = DigitalHomodyne(label='M2', boxCarStart=150, boxCarStop=600, IFfreq=39.2e6, samplingRate=250e6, channel=2)
    testFilter3 = Correlator(label='M12')
    filterDict = {'M1':testFilter1, 'M2':testFilter2, 'M12':testFilter3}

    testLib = MeasFilterLibrary(libFile='MeasFilterLibrary.json', filterDict=filterDict)

    with enaml.imports():
        from MeasFiltersViews import MeasFilterManagerWindow

    app = QtApplication()
    view = MeasFilterManagerWindow(filterLib=testLib)
    view.show()

    app.start()
<|endoftext|>


<commit_before><commit_msg>imporoved logout test<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    Copyright (C) 2014 Thibaut Lapierre <root@epheo.eu>. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import docker
import json
from shaddock import model

class Image(object):
    def __init__(self, name, docker_host, docker_version):
        self.name = name
        self.containerconfig = model.ContainerConfig(self.name)
        self.docker_host = docker_host
        self.docker_version = docker_version
        self.dockerapi = docker.Client(base_url=self.docker_host,
                                       version=str(self.docker_version),
                                       timeout=10)

    def build(self, nocache):
        if self.containerconfig.tag is not None:
            for line in self.dockerapi.build(path=self.containerconfig.path,
                                             tag=self.containerconfig.tag,
                                             nocache=nocache):
                jsonstream = json.loads(line.decode())
                stream = jsonstream.get('stream')
                error = jsonstream.get('error')
                if error is not None:
                    print(error)
                if stream is not None:
                    print(stream)
        else:
            print("Unrecognized service name")

    def create(self):
        c_id = self.dockerapi.create_container(
            image=self.containerconfig.tag,
            name=self.name,
            detach=False,
            ports=self.containerconfig.ports,
            environment=model.get_vars_dict(),
            volumes=self.containerconfig.volumes,
            hostname=self.containerconfig.name)

        return c_id

class Container(object):
    def __init__(self, service_name, docker_host, docker_version):
        self.name = service_name
        self.containerconfig = model.ContainerConfig(self.name)
        self.tag = self.containerconfig.tag
        if self.containerconfig.privileged:
            self.privileged = self.containerconfig.privileged
        else:
            self.privileged = None
        if self.containerconfig.network_mode:
            self.network_mode = self.containerconfig.network_mode
        else:
            self.network_mode = 'bridge'
        self.docker_host = docker_host
        self.docker_version = docker_version
        self.dockerapi = docker.Client(base_url=self.docker_host,
                                       version=str(self.docker_version),
                                       timeout=10)
        info = self.get_info()
        self.id = info.get('id')
        self.ip = info.get('ip')
        self.hostname = info.get('hostname')
        self.started = info.get('started')
        self.created = info.get('created')

    def start(self):
        if self.started is False and self.created is True:
            print('Starting %s ...' % self.tag)
            self.dockerapi.start(
                    container=self.id,
                    binds=self.containerconfig.binds,
                    port_bindings=self.containerconfig.port_bindings,
                    privileged=self.privileged,
                    network_mode=self.network_mode)
        elif self.created is False:
            print('Creating image %s ...' % self.tag)
            image = Image(self.name, self.docker_host, self.docker_version)
            c_id = image.create()
            if c_id:
                print('Starting container ...')
                self.dockerapi.start(
                    container=c_id,
                    binds=self.containerconfig.binds,
                    port_bindings=self.containerconfig.port_bindings,
                    privileged=self.privileged,
                    network_mode=self.network_mode)

        return True

    def stop(self):
        if self.started is True:
            print('Stopping container %s ...' % self.tag)
            self.dockerapi.stop(self.id)

    def remove(self):
        self.stop()
        if self.created is True:
            print('Removing container %s ...' % self.id)
            self.dockerapi.remove_container(self.id)

    def restart(self):
        self.dockerapi.restart(self.id)

    def return_logs(self):
        if self.containerconfig.tag is not None:
            for line in self.dockerapi.logs(
                                       container=self.id,
                                       stderr=True,
                                       stdout=True,
                                       stream=True):
                try:
                    print(line)
                except (KeyboardInterrupt, SystemExit):
                    return True

    def pull(self):
        for line in self.dockerapi.pull(self.tag, stream=True):
            print(json.dumps(json.loads(line), indent=4))

    def get_info(self):
        info = {}
        info['id'] = None
        info['ip'] = None
        info['hostname'] = None
        info['started'] = False
        info['created'] = False
        containers_list = self.dockerapi.containers(all=True)
        if containers_list:
            try:
                c_id = [item['Id'] for item in containers_list
                        if self.tag in item['Image']][0]
            except IndexError:
                c_id = None

            if c_id:
                container_info = self.dockerapi.inspect_container(c_id)
                config = container_info['Config']
                network = container_info['NetworkSettings']
                info['id'] = c_id
                info['ip'] = network['IPAddress']
                info['hostname'] = config['Hostname']
                info['created'] = True
                if info.get('ip'):
                    info['started'] = True
        return info
<commit_msg>Change output of logs generator to str<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

#    Copyright (C) 2014 Thibaut Lapierre <root@epheo.eu>. All Rights Reserved.
#
#    Licensed under the Apache License, Version 2.0 (the "License"); you may
#    not use this file except in compliance with the License. You may obtain
#    a copy of the License at
#
#         http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
#    License for the specific language governing permissions and limitations
#    under the License.

import docker
import json
from shaddock import model

class Image(object):
    def __init__(self, name, docker_host, docker_version):
        self.name = name
        self.containerconfig = model.ContainerConfig(self.name)
        self.docker_host = docker_host
        self.docker_version = docker_version
        self.dockerapi = docker.Client(base_url=self.docker_host,
                                       version=str(self.docker_version),
                                       timeout=10)

    def build(self, nocache):
        if self.containerconfig.tag is not None:
            for line in self.dockerapi.build(path=self.containerconfig.path,
                                             tag=self.containerconfig.tag,
                                             nocache=nocache):
                jsonstream = json.loads(line.decode())
                stream = jsonstream.get('stream')
                error = jsonstream.get('error')
                if error is not None:
                    print(error)
                if stream is not None:
                    print(stream)
        else:
            print("Unrecognized service name")

    def create(self):
        c_id = self.dockerapi.create_container(
            image=self.containerconfig.tag,
            name=self.name,
            detach=False,
            ports=self.containerconfig.ports,
            environment=model.get_vars_dict(),
            volumes=self.containerconfig.volumes,
            hostname=self.containerconfig.name)

        return c_id

class Container(object):
    def __init__(self, service_name, docker_host, docker_version):
        self.name = service_name
        self.containerconfig = model.ContainerConfig(self.name)
        self.tag = self.containerconfig.tag
        if self.containerconfig.privileged:
            self.privileged = self.containerconfig.privileged
        else:
            self.privileged = None
        if self.containerconfig.network_mode:
            self.network_mode = self.containerconfig.network_mode
        else:
            self.network_mode = 'bridge'
        self.docker_host = docker_host
        self.docker_version = docker_version
        self.dockerapi = docker.Client(base_url=self.docker_host,
                                       version=str(self.docker_version),
                                       timeout=10)
        info = self.get_info()
        self.id = info.get('id')
        self.ip = info.get('ip')
        self.hostname = info.get('hostname')
        self.started = info.get('started')
        self.created = info.get('created')

    def start(self):
        if self.started is False and self.created is True:
            print('Starting %s ...' % self.tag)
            self.dockerapi.start(
                    container=self.id,
                    binds=self.containerconfig.binds,
                    port_bindings=self.containerconfig.port_bindings,
                    privileged=self.privileged,
                    network_mode=self.network_mode)
        elif self.created is False:
            print('Creating image %s ...' % self.tag)
            image = Image(self.name, self.docker_host, self.docker_version)
            c_id = image.create()
            if c_id:
                print('Starting container ...')
                self.dockerapi.start(
                    container=c_id,
                    binds=self.containerconfig.binds,
                    port_bindings=self.containerconfig.port_bindings,
                    privileged=self.privileged,
                    network_mode=self.network_mode)

        return True

    def stop(self):
        if self.started is True:
            print('Stopping container %s ...' % self.tag)
            self.dockerapi.stop(self.id)

    def remove(self):
        self.stop()
        if self.created is True:
            print('Removing container %s ...' % self.id)
            self.dockerapi.remove_container(self.id)

    def restart(self):
        self.dockerapi.restart(self.id)

    def return_logs(self):
        if self.containerconfig.tag is not None:
            for line in self.dockerapi.logs(
                                       container=self.id,
                                       stderr=True,
                                       stdout=True,
                                       stream=True):
                try:
                    print(str(line))
                except (KeyboardInterrupt, SystemExit):
                    return True

    def pull(self):
        for line in self.dockerapi.pull(self.tag, stream=True):
            print(json.dumps(json.loads(line), indent=4))

    def get_info(self):
        info = {}
        info['id'] = None
        info['ip'] = None
        info['hostname'] = None
        info['started'] = False
        info['created'] = False
        containers_list = self.dockerapi.containers(all=True)
        if containers_list:
            try:
                c_id = [item['Id'] for item in containers_list
                        if self.tag in item['Image']][0]
            except IndexError:
                c_id = None

            if c_id:
                container_info = self.dockerapi.inspect_container(c_id)
                config = container_info['Config']
                network = container_info['NetworkSettings']
                info['id'] = c_id
                info['ip'] = network['IPAddress']
                info['hostname'] = config['Hostname']
                info['created'] = True
                if info.get('ip'):
                    info['started'] = True
        return info
<|endoftext|>


<commit_before><commit_msg>test main only uses argparse now, no more subprocess<commit_after><|endoftext|>


<commit_before>from __future__ import division
import pytest
import popmodel as pm
from pkg_resources import resource_filename
from collections import Mapping
from copy import deepcopy

@pytest.fixture(scope='session')
def hpar():
    hpath = resource_filename('popmodel', 'data/hitran_sample.par')
    return pm.loadhitran.processhitran(hpath)

# Use parameters_template YAML file included with package as base set of
# parameters. Toggle settings in deepcopies of par (deepcopy needed for nested
# dict) in tests as needed.
@pytest.fixture(scope='session')
def par():
    yamlpath = resource_filename('popmodel','data/parameters_template.yaml')
    return pm.importyaml(yamlpath)

# KineticsRun instance without running solveode
@pytest.fixture(scope='session')
def k(hpar,par):
    return pm.KineticsRun(hpar,**par)

# KineticsRun instance that has had solveode() run.
@pytest.fixture(scope='session')
def k_solved(hpar, par):
    k_solved = pm.KineticsRun(hpar, **par)
    k_solved.solveode()
    return k_solved

# KineticsRun instance with dosweep = True
@pytest.fixture(scope='session')
def k_sweep(hpar, par):
    par_sweep = deepcopy(par)
    par_sweep['sweep']['dosweep'] = True
    k_sweep = pm.KineticsRun(hpar, **par_sweep)
    k_sweep.solveode()
    return k_sweep

# KineticsRun instance set up for UV (vibrationless transition) without IR
@pytest.fixture(scope='session')
def k_uvonly(hpar, par):
    par_uvonly = deepcopy(par)
    uvonly_reqs = {'uvline': {'vib': '00'}, 'odepar': {'withoutUV': False,
                                                       'withoutIR': True}}
    _nestedupdate(par_uvonly, uvonly_reqs)
    k_uvonly = pm.KineticsRun(hpar, **par_uvonly)
    k_uvonly.solveode()
    return k_uvonly

def test_KineticsRun_instance_has_attributes(k):
    assert hasattr(k, 'detcell')
    assert hasattr(k, 'irlaser')
    assert hasattr(k, 'uvlaser')
    assert hasattr(k, 'odepar')
    assert hasattr(k, 'irline')
    assert hasattr(k, 'uvline')
    assert hasattr(k, 'rates')

def test_raise_error_popsfigure_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.popsfigure()

def test_raise_error_popseries_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.popseries('b')

def test_raise_error_calcfluor_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.calcfluor()

def test_raise_error_vslaserfigure_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.vslaserfigure(lambda x: 'dummy arg')

def test_raise_error_savecsv_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.savecsv('does_not_reach_output')

def test_raise_error_saveoutput_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.saveoutput('does_not_reach_output')

@pytest.mark.mpl_image_compare
def test_popsfigure_plots_suite_of_subpop_popcodes(k_solved):
    return k_solved.popsfigure(subpop=['ahd','bsl','clp','dda','b'])

@pytest.mark.mpl_image_compare
def test_popsfigure_default_subpop(k_solved):
    return k_solved.popsfigure()

@pytest.mark.mpl_image_compare
def test_popsfigure_integrate_uvlaser_only(k_uvonly):
    return k_uvonly.popsfigure(subpop=['b', 'c'])

@pytest.mark.mpl_image_compare
def test_vslaserfigure_defaults(k_sweep):
    return k_sweep.vslaserfigure(k_sweep.popseries('b'))

@pytest.mark.mpl_image_compare
def test_absfigure_defaults(k_sweep):
    return k_sweep.absfigure()

def test_calcfluor_returns_value(k_solved):
    assert k_solved.calcfluor()

def test_calcfluor_uvonly_returns_positive_value(k_uvonly):
    assert (k_uvonly.calcfluor() > 0)

@pytest.mark.parametrize("par_laser_setup", [
    {'uvline': {'vib': '00'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '10'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '11'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '01'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'odepar': {'withoutUV': True, 'withoutIR': True}},
    {'odepar': {'withoutUV': True, 'withoutIR': False}},
    ])
def test_KineticsRun_init_toggle_laser_setup(hpar, par, par_laser_setup):
    par_lasers = deepcopy(par)
    _nestedupdate(par_lasers, par_laser_setup)
    k_lasers = pm.KineticsRun(hpar, **par_lasers)
    assert hasattr(k_lasers, 'levels')

@pytest.mark.parametrize("par_laser_bad", [
    {'uvline': {'vib': '20'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '20'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '02'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '01'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    ])
def test_KineticsRun_init_toggle_laser_setup_ValueErrors(hpar, par,
                                                         par_laser_bad):
    par_lasers = deepcopy(par)
    _nestedupdate(par_lasers, par_laser_bad)
    with pytest.raises(ValueError):
        k = pm.KineticsRun(hpar, **par_lasers)

# def test_unicode_input():
#     \u1E98\u03B5\u1E2F\u0491\u1E13\u2624\u03B7\u2118\u028A\u2602

# Update nested dictionary like `par` for tests with parametrized values,
# see https://stackoverflow.com/a/3233356
def _nestedupdate(d, u):
    for k,v in u.items():
        if isinstance(v, Mapping):
            r = _nestedupdate(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d
<commit_msg>comment out crashing k_sweep, fix fixtures<commit_after>from __future__ import division
import pytest
import popmodel as pm
from pkg_resources import resource_filename
from collections import Mapping
from copy import deepcopy

# Use parameters_template YAML file included with package as base set of
# parameters. Toggle settings in deepcopies of par (deepcopy needed for nested
# dict) in tests as needed.
@pytest.fixture(scope='module')
def par():
    yamlpath = resource_filename('popmodel','data/parameters_template.yaml')
    return pm.importyaml(yamlpath)

# KineticsRun instance without running solveode
@pytest.fixture(scope='module')
def k(hpar,par):
    return pm.KineticsRun(hpar,**par)

# KineticsRun instance that has had solveode() run.
@pytest.fixture(scope='module')
def k_solved(hpar, par):
    k_solved = pm.KineticsRun(hpar, **par)
    k_solved.solveode()
    return k_solved

# # KineticsRun instance with dosweep = True
# @pytest.fixture(scope='module')
# def k_sweep(hpar, par):
#     par_sweep = deepcopy(par)
#     par_sweep['sweep']['dosweep'] = True
#     k_sweep = pm.KineticsRun(hpar, **par_sweep)
#     k_sweep.solveode()
#     return k_sweep

# KineticsRun instance set up for UV (vibrationless transition) without IR
@pytest.fixture(scope='module')
def k_uvonly(hpar, par):
    par_uvonly = deepcopy(par)
    uvonly_reqs = {'uvline': {'vib': '00'}, 'odepar': {'withoutUV': False,
                                                       'withoutIR': True}}
    _nestedupdate(par_uvonly, uvonly_reqs)
    k_uvonly = pm.KineticsRun(hpar, **par_uvonly)
    k_uvonly.solveode()
    return k_uvonly

def test_KineticsRun_instance_has_attributes(k):
    assert hasattr(k, 'detcell')
    assert hasattr(k, 'irlaser')
    assert hasattr(k, 'uvlaser')
    assert hasattr(k, 'odepar')
    assert hasattr(k, 'irline')
    assert hasattr(k, 'uvline')
    assert hasattr(k, 'rates')

def test_raise_error_popsfigure_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.popsfigure()

def test_raise_error_popseries_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.popseries('b')

def test_raise_error_calcfluor_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.calcfluor()

def test_raise_error_vslaserfigure_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.vslaserfigure(lambda x: 'dummy arg')

def test_raise_error_savecsv_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.savecsv('does_not_reach_output')

def test_raise_error_saveoutput_without_pop_full(k):
    with pytest.raises(AttributeError):
        k.saveoutput('does_not_reach_output')

@pytest.mark.mpl_image_compare
def test_popsfigure_plots_suite_of_subpop_popcodes(k_solved):
    return k_solved.popsfigure(subpop=['ahd','bsl','clp','dda','b'])

@pytest.mark.mpl_image_compare
def test_popsfigure_default_subpop(k_solved):
    return k_solved.popsfigure()

@pytest.mark.mpl_image_compare
def test_popsfigure_integrate_uvlaser_only(k_uvonly):
    return k_uvonly.popsfigure(subpop=['b', 'c'])

# @pytest.mark.mpl_image_compare
# def test_vslaserfigure_defaults(k_sweep):
#     return k_sweep.vslaserfigure(k_sweep.popseries('b'))

# @pytest.mark.mpl_image_compare
# def test_absfigure_defaults(k_sweep):
#     return k_sweep.absfigure()

def test_calcfluor_returns_value(k_solved):
    assert k_solved.calcfluor()

def test_calcfluor_uvonly_returns_positive_value(k_uvonly):
    assert (k_uvonly.calcfluor() > 0)

@pytest.mark.parametrize("par_laser_setup", [
    {'uvline': {'vib': '00'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '10'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '11'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '01'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'odepar': {'withoutUV': True, 'withoutIR': True}},
    {'odepar': {'withoutUV': True, 'withoutIR': False}},
    ])
def test_KineticsRun_init_toggle_laser_setup(hpar, par, par_laser_setup):
    par_lasers = deepcopy(par)
    _nestedupdate(par_lasers, par_laser_setup)
    k_lasers = pm.KineticsRun(hpar, **par_lasers)
    assert hasattr(k_lasers, 'levels')

@pytest.mark.parametrize("par_laser_bad", [
    {'uvline': {'vib': '20'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    {'uvline': {'vib': '20'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '02'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': False}},
    {'uvline': {'vib': '01'}, 'odepar': {'withoutUV': False,
                                         'withoutIR': True}},
    ])
def test_KineticsRun_init_toggle_laser_setup_ValueErrors(hpar, par,
                                                         par_laser_bad):
    par_lasers = deepcopy(par)
    _nestedupdate(par_lasers, par_laser_bad)
    with pytest.raises(ValueError):
        k = pm.KineticsRun(hpar, **par_lasers)

# def test_unicode_input():
#     \u1E98\u03B5\u1E2F\u0491\u1E13\u2624\u03B7\u2118\u028A\u2602

# Update nested dictionary like `par` for tests with parametrized values,
# see https://stackoverflow.com/a/3233356
def _nestedupdate(d, u):
    for k,v in u.items():
        if isinstance(v, Mapping):
            r = _nestedupdate(d.get(k, {}), v)
            d[k] = r
        else:
            d[k] = u[k]
    return d
<|endoftext|>