<commit_before>import difflib
import string

import re

from .string_utils import harmonize

STOP_WORDS = (
    "a|an|and|as|at|but|by|en|etc|for|if|in|is|of|on|or|the|to|v\.?|via|"
    "vs\.?|et|al|appellants?|defendants?|administrator|"
    "plaintiffs?|error|others|against|ex|parte|complainants?|original|"
    "claimants?|devisee|executrix|executor|comm(issione)?\'?r"
)
STOP_WORDS_RE = re.compile(r"^(%s)$" % STOP_WORDS)

def normalize_phrase(phrase):
    """Clean up words or phrases before sending them to be compared.

     - Harmonize things like United States, USA, etc.
     - Normalize to lower case.
     - Remove punctuation.
     - Remove stop words.
    """
    phrase = harmonize(phrase)
    phrase = phrase.lower()

    # strip punctuation
    exclude = set(string.punctuation)
    phrase = "".join(ch for ch in phrase if ch not in exclude)

    words = re.split("[\t ]", phrase)
    result = []
    for word in words:
        word = STOP_WORDS_RE.sub("", word)
        result.append(word)
    return "".join(result)

def get_closest_match_index(word, possibilities):
    """Find the string that is most similar to the target string. Uses difflib's
    SequenceMatcher under the covers.

    :param word: The string to match to.
    :param possibilities: A list of possible matches.
    :return: The index of the closest matching possibility, if any. Else, return
    None.
    """
    word = normalize_phrase(word)
    possibilities = [normalize_phrase(x) for x in possibilities]
    try:
        match = difflib.get_close_matches(
            word, possibilities, n=1, cutoff=0.1
        )[0]
    except IndexError:
        # No good matches.
        return None
    return possibilities.index(match)
<commit_msg>fix(diff_tools): Remove apostrophe from STOP WORDS<commit_after>import difflib
import string

import re

from .string_utils import harmonize

STOP_WORDS = (
    "a|an|and|as|at|but|by|en|etc|for|if|in|is|of|on|or|the|to|v\.?|via|"
    "vs\.?|et|al|appellants?|defendants?|administrator|"
    "plaintiffs?|error|others|against|ex|parte|complainants?|original|"
    "claimants?|devisee|executrix|executor|comm(issione)?r"
)
STOP_WORDS_RE = re.compile(r"^(%s)$" % STOP_WORDS)

def normalize_phrase(phrase):
    """Clean up words or phrases before sending them to be compared.

     - Harmonize things like United States, USA, etc.
     - Normalize to lower case.
     - Remove punctuation.
     - Remove stop words.
    """
    phrase = harmonize(phrase)
    phrase = phrase.lower()

    # strip punctuation
    exclude = set(string.punctuation)
    phrase = "".join(ch for ch in phrase if ch not in exclude)

    words = re.split("[\t ]", phrase)
    result = []
    for word in words:
        word = STOP_WORDS_RE.sub("", word)
        result.append(word)
    return "".join(result)

def get_closest_match_index(word, possibilities):
    """Find the string that is most similar to the target string. Uses difflib's
    SequenceMatcher under the covers.

    :param word: The string to match to.
    :param possibilities: A list of possible matches.
    :return: The index of the closest matching possibility, if any. Else, return
    None.
    """
    word = normalize_phrase(word)
    possibilities = [normalize_phrase(x) for x in possibilities]
    try:
        match = difflib.get_close_matches(
            word, possibilities, n=1, cutoff=0.1
        )[0]
    except IndexError:
        # No good matches.
        return None
    return possibilities.index(match)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
import os
import sys
import warnings

from requests.exceptions import ConnectionError
from requests.models import Request, Response

from .exceptions import SlownessException

WARN_SLOW_SRAPERS = 'TRAVIS' in os.environ
# FIXME: when testing, should we only find the first warning
# or provide a warning for each slow scraper?
warnings.simplefilter("always")

class SlownessWarning(UserWarning):
    pass

class TooSlowWarning(SlownessWarning):
    pass

class MockRequest(Request):
    def __init__(self, url=None):
        super(Request, self).__init__()
        self.url = url
        #: Resulting :class:`HTTPError` of request, if one occurred.
        self.error = None

        #: Encoding to decode with when accessing r.content.
        self.encoding = None

        #: The :class:`Request <Request>` that created the Response.
        self.request = self

    def get(self):
        r = Response()
        try:
            with open(self.url, mode='rb') as stream:
                r._content = stream.read()
                #: Integer Code of responded HTTP Status.
                r.status_code = 200
                if self.url.endswith('json'):
                    r.headers['content-type'] = 'application/json'
        except IOError as e:
            r.status_code = 404
            raise ConnectionError(e)

        r._content_consumed = True

        #: Final URL location of Response.
        r.url = self.url

        # Return the response.
        return r

def warn_or_crash_slow_parser(duration, warn_duration=1, max_duration=15):
    msg = ''
    if duration > max_duration:
        if sys.gettrace() is None and not WARN_SLOW_SRAPERS:
            # Only do this if we're not debugging. Debuggers make things slower
            # and breakpoints make things stop.
            raise SlownessException(
                "This scraper took {duration}s to test, which is more than "
                "the allowed speed of {max_duration}s. Please speed it up for "
                "tests to pass.".format(duration=duration,
                                        max_duration=max_duration)
            )
        elif WARN_SLOW_SRAPERS:
            msg = ' - WARNING: SLOW SCRAPER'
            wmsg = "WARNING: VERY SLOW SCRAPER (potential Error)"
            warnings.warn(wmsg, TooSlowWarning)
    elif duration > warn_duration:
        msg = ' - WARNING: SLOW SCRAPER'
        warnings.warn('WARNING: SLOW SCRAPER', SlownessWarning)
    else:
        msg = ''
    return msg
<commit_msg>remove fixme comment<commit_after># -*- coding: utf-8 -*-
import os
import sys
import warnings

from requests.exceptions import ConnectionError
from requests.models import Request, Response

from .exceptions import SlownessException

WARN_SLOW_SRAPERS = 'TRAVIS' in os.environ
warnings.simplefilter("always")

class SlownessWarning(UserWarning):
    pass

class TooSlowWarning(SlownessWarning):
    pass

class MockRequest(Request):
    def __init__(self, url=None):
        super(Request, self).__init__()
        self.url = url
        #: Resulting :class:`HTTPError` of request, if one occurred.
        self.error = None

        #: Encoding to decode with when accessing r.content.
        self.encoding = None

        #: The :class:`Request <Request>` that created the Response.
        self.request = self

    def get(self):
        r = Response()
        try:
            with open(self.url, mode='rb') as stream:
                r._content = stream.read()
                #: Integer Code of responded HTTP Status.
                r.status_code = 200
                if self.url.endswith('json'):
                    r.headers['content-type'] = 'application/json'
        except IOError as e:
            r.status_code = 404
            raise ConnectionError(e)

        r._content_consumed = True

        #: Final URL location of Response.
        r.url = self.url

        # Return the response.
        return r

def warn_or_crash_slow_parser(duration, warn_duration=1, max_duration=15):
    msg = ''
    if duration > max_duration:
        if sys.gettrace() is None and not WARN_SLOW_SRAPERS:
            # Only do this if we're not debugging. Debuggers make things slower
            # and breakpoints make things stop.
            raise SlownessException(
                "This scraper took {duration}s to test, which is more than "
                "the allowed speed of {max_duration}s. Please speed it up for "
                "tests to pass.".format(duration=duration,
                                        max_duration=max_duration)
            )
        elif WARN_SLOW_SRAPERS:
            msg = ' - WARNING: SLOW SCRAPER'
            wmsg = "WARNING: VERY SLOW SCRAPER (potential Error)"
            warnings.warn(wmsg, TooSlowWarning)
    elif duration > warn_duration:
        msg = ' - WARNING: SLOW SCRAPER'
        warnings.warn('WARNING: SLOW SCRAPER', SlownessWarning)
    else:
        msg = ''
    return msg
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""A Web interface to beets."""
from __future__ import division, absolute_import, print_function

from beets.plugins import BeetsPlugin
from beets import ui
from beets import util
import beets.library
import flask
from flask import g
from werkzeug.routing import BaseConverter, PathConverter
import os
import json

# Utilities.

def _rep(obj, expand=False):
    """Get a flat -- i.e., JSON-ish -- representation of a beets Item or
    Album object. For Albums, `expand` dictates whether tracks are
    included.
    """
    out = dict(obj)

    if isinstance(obj, beets.library.Item):
        if app.config['exclude_paths_from_items']:
            del out['path']
        else:
            out['path'] = out['path'].decode('utf-8')

        # Get the size (in bytes) of the backing file. This is useful
        # for the Tomahawk resolver API.
        try:
            out['size'] = os.path.getsize(util.syspath(obj.path))
        except OSError:
            out['size'] = 0

        return out

    elif isinstance(obj, beets.library.Album):
        del out['artpath']
        if expand:
            out['items'] = [_rep(item) for item in obj.items()]
        return out

def json_generator(items, root, expand=False):
    """Generator that dumps list of beets Items or Albums as JSON

    :param root:  root key for JSON
    :param items: list of :class:`Item` or :class:`Album` to dump
    :param expand: If true every :class:`Album` contains its items in the json
                   representation
    :returns:     generator that yields strings
    """
    yield '{"%s":[' % root
    first = True
    for item in items:
        if first:
            first = False
        else:
            yield ','
        yield json.dumps(_rep(item, expand=expand))
    yield ']}'

def is_expand():
    """Returns whether the current request is for an expanded response."""

    return flask.request.args.get('expand') is not None

def resource(name):
    """Decorates a function to handle RESTful HTTP requests for a resource.
    """
    def make_responder(retriever):
        def responder(ids):
            entities = [retriever(id) for id in ids]
            entities = [entity for entity in entities if entity]

            if len(entities) == 1:
                return flask.jsonify(_rep(entities[0], expand=is_expand()))
            elif entities:
                return app.response_class(
                    json_generator(entities, root=name),
                    mimetype='application/json'
                )
            else:
                return flask.abort(404)
        responder.__name__ = 'get_{0}'.format(name)
        return responder
    return make_responder

def resource_query(name):
    """Decorates a function to handle RESTful HTTP queries for resources.
    """
    def make_responder(query_func):
        def responder(queries):
            return app.response_class(
                json_generator(
                    query_func(queries),
                    root='results', expand=is_expand()
                ),
                mimetype='application/json'
            )
        responder.__name__ = 'query_{0}'.format(name)
        return responder
    return make_responder

def resource_list(name):
    """Decorates a function to handle RESTful HTTP request for a list of
    resources.
    """
    def make_responder(list_all):
        def responder():
            return app.response_class(
                json_generator(list_all(), root=name, expand=is_expand()),
                mimetype='application/json'
            )
        responder.__name__ = 'all_{0}'.format(name)
        return responder
    return make_responder

def _get_unique_table_field_values(model, field, sort_field):
    """ retrieve all unique values belonging to a key from a model """
    if field not in model.all_keys() or sort_field not in model.all_keys():
        raise KeyError
    with g.lib.transaction() as tx:
        rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"'
                        .format(field, model._table, sort_field))
    return [row[0] for row in rows]

class IdListConverter(BaseConverter):
    """Converts comma separated lists of ids in urls to integer lists.
    """

    def to_python(self, value):
        ids = []
        for id in value.split(','):
            try:
                ids.append(int(id))
            except ValueError:
                pass
        return ids

    def to_url(self, value):
        return ','.join(value)

class QueryConverter(PathConverter):
    """Converts slash separated lists of queries in the url to string list.
    """

    def to_python(self, value):
        return value.split('/')

    def to_url(self, value):
        return ','.join(value)

# Flask setup.

app = flask.Flask(__name__)
app.url_map.converters['idlist'] = IdListConverter
app.url_map.converters['query'] = QueryConverter

@app.before_request
def before_request():
    g.lib = app.config['lib']

# Items.

@app.route('/item/<idlist:ids>')
@resource('items')
def get_item(id):
    return g.lib.get_item(id)

@app.route('/item/')
@app.route('/item/query/')
@resource_list('items')
def all_items():
    return g.lib.items()

@app.route('/item/<int:item_id>/file')
def item_file(item_id):
    item = g.lib.get_item(item_id)
    response = flask.send_file(
        util.py3_path(item.path),
        as_attachment=True,
        attachment_filename=os.path.basename(util.py3_path(item.path)),
    )
    response.headers['Content-Length'] = os.path.getsize(item.path)
    return response

@app.route('/item/query/<query:queries>')
@resource_query('items')
def item_query(queries):
    return g.lib.items(queries)

@app.route('/item/at_path/<path:path>')
def item_at_path(path):
    try:
        return flask.jsonify(_rep(beets.library.Item.from_path('/' + path)))
    except beets.library.ReadError:
        return flask.abort(404)

@app.route('/item/values/<string:key>')
def item_unique_field_values(key):
    sort_key = flask.request.args.get('sort_key', key)
    try:
        values = _get_unique_table_field_values(beets.library.Item, key,
                                                sort_key)
    except KeyError:
        return flask.abort(404)
    return flask.jsonify(values=values)

# Albums.

@app.route('/album/<idlist:ids>')
@resource('albums')
def get_album(id):
    return g.lib.get_album(id)

@app.route('/album/')
@app.route('/album/query/')
@resource_list('albums')
def all_albums():
    return g.lib.albums()

@app.route('/album/query/<query:queries>')
@resource_query('albums')
def album_query(queries):
    return g.lib.albums(queries)

@app.route('/album/<int:album_id>/art')
def album_art(album_id):
    album = g.lib.get_album(album_id)
    if album.artpath:
        return flask.send_file(album.artpath)
    else:
        return flask.abort(404)

@app.route('/album/values/<string:key>')
def album_unique_field_values(key):
    sort_key = flask.request.args.get('sort_key', key)
    try:
        values = _get_unique_table_field_values(beets.library.Album, key,
                                                sort_key)
    except KeyError:
        return flask.abort(404)
    return flask.jsonify(values=values)

# Artists.

@app.route('/artist/')
def all_artists():
    with g.lib.transaction() as tx:
        rows = tx.query("SELECT DISTINCT albumartist FROM albums")
    all_artists = [row[0] for row in rows]
    return flask.jsonify(artist_names=all_artists)

# Library information.

@app.route('/stats')
def stats():
    with g.lib.transaction() as tx:
        item_rows = tx.query("SELECT COUNT(*) FROM items")
        album_rows = tx.query("SELECT COUNT(*) FROM albums")
    return flask.jsonify({
        'items': item_rows[0][0],
        'albums': album_rows[0][0],
    })

# UI.

@app.route('/')
def home():
    return flask.render_template('index.html')

# Plugin hook.

class WebPlugin(BeetsPlugin):
    def __init__(self):
        super(WebPlugin, self).__init__()
        self.config.add({
            'host': u'127.0.0.1',
            'port': 8337,
            'cors': '',
            'exclude_paths_from_items': True,
        })

    def commands(self):
        cmd = ui.Subcommand('web', help=u'start a Web interface')
        cmd.parser.add_option(u'-d', u'--debug', action='store_true',
                              default=False, help=u'debug mode')

        def func(lib, opts, args):
            args = ui.decargs(args)
            if args:
                self.config['host'] = args.pop(0)
            if args:
                self.config['port'] = int(args.pop(0))

            app.config['lib'] = lib
            # Normalizes json output
            app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False

            app.config['exclude_paths_from_items'] = (
                self.config.get('exclude_paths_from_items', True))

            # Enable CORS if required.
            if self.config['cors']:
                self._log.info(u'Enabling CORS with origin: {0}',
                               self.config['cors'])
                from flask.ext.cors import CORS
                app.config['CORS_ALLOW_HEADERS'] = "Content-Type"
                app.config['CORS_RESOURCES'] = {
                    r"/*": {"origins": self.config['cors'].get(str)}
                }
                CORS(app)
            # Start the web application.
            app.run(host=self.config['host'].as_str(),
                    port=self.config['port'].get(int),
                    debug=opts.debug, threaded=True)
        cmd.func = func
        return [cmd]
<commit_msg>Fix tests I broke<commit_after># -*- coding: utf-8 -*-
# This file is part of beets.
# Copyright 2016, Adrian Sampson.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.

"""A Web interface to beets."""
from __future__ import division, absolute_import, print_function

from beets.plugins import BeetsPlugin
from beets import ui
from beets import util
import beets.library
import flask
from flask import g
from werkzeug.routing import BaseConverter, PathConverter
import os
import json

# Utilities.

def _rep(obj, expand=False):
    """Get a flat -- i.e., JSON-ish -- representation of a beets Item or
    Album object. For Albums, `expand` dictates whether tracks are
    included.
    """
    out = dict(obj)

    if isinstance(obj, beets.library.Item):
        if app.config.get('EXCLUDE_PATHS_FROM_ITEMS', True):
            del out['path']
        else:
            out['path'] = out['path'].decode('utf-8')

        # Get the size (in bytes) of the backing file. This is useful
        # for the Tomahawk resolver API.
        try:
            out['size'] = os.path.getsize(util.syspath(obj.path))
        except OSError:
            out['size'] = 0

        return out

    elif isinstance(obj, beets.library.Album):
        del out['artpath']
        if expand:
            out['items'] = [_rep(item) for item in obj.items()]
        return out

def json_generator(items, root, expand=False):
    """Generator that dumps list of beets Items or Albums as JSON

    :param root:  root key for JSON
    :param items: list of :class:`Item` or :class:`Album` to dump
    :param expand: If true every :class:`Album` contains its items in the json
                   representation
    :returns:     generator that yields strings
    """
    yield '{"%s":[' % root
    first = True
    for item in items:
        if first:
            first = False
        else:
            yield ','
        yield json.dumps(_rep(item, expand=expand))
    yield ']}'

def is_expand():
    """Returns whether the current request is for an expanded response."""

    return flask.request.args.get('expand') is not None

def resource(name):
    """Decorates a function to handle RESTful HTTP requests for a resource.
    """
    def make_responder(retriever):
        def responder(ids):
            entities = [retriever(id) for id in ids]
            entities = [entity for entity in entities if entity]

            if len(entities) == 1:
                return flask.jsonify(_rep(entities[0], expand=is_expand()))
            elif entities:
                return app.response_class(
                    json_generator(entities, root=name),
                    mimetype='application/json'
                )
            else:
                return flask.abort(404)
        responder.__name__ = 'get_{0}'.format(name)
        return responder
    return make_responder

def resource_query(name):
    """Decorates a function to handle RESTful HTTP queries for resources.
    """
    def make_responder(query_func):
        def responder(queries):
            return app.response_class(
                json_generator(
                    query_func(queries),
                    root='results', expand=is_expand()
                ),
                mimetype='application/json'
            )
        responder.__name__ = 'query_{0}'.format(name)
        return responder
    return make_responder

def resource_list(name):
    """Decorates a function to handle RESTful HTTP request for a list of
    resources.
    """
    def make_responder(list_all):
        def responder():
            return app.response_class(
                json_generator(list_all(), root=name, expand=is_expand()),
                mimetype='application/json'
            )
        responder.__name__ = 'all_{0}'.format(name)
        return responder
    return make_responder

def _get_unique_table_field_values(model, field, sort_field):
    """ retrieve all unique values belonging to a key from a model """
    if field not in model.all_keys() or sort_field not in model.all_keys():
        raise KeyError
    with g.lib.transaction() as tx:
        rows = tx.query('SELECT DISTINCT "{0}" FROM "{1}" ORDER BY "{2}"'
                        .format(field, model._table, sort_field))
    return [row[0] for row in rows]

class IdListConverter(BaseConverter):
    """Converts comma separated lists of ids in urls to integer lists.
    """

    def to_python(self, value):
        ids = []
        for id in value.split(','):
            try:
                ids.append(int(id))
            except ValueError:
                pass
        return ids

    def to_url(self, value):
        return ','.join(value)

class QueryConverter(PathConverter):
    """Converts slash separated lists of queries in the url to string list.
    """

    def to_python(self, value):
        return value.split('/')

    def to_url(self, value):
        return ','.join(value)

# Flask setup.

app = flask.Flask(__name__)
app.url_map.converters['idlist'] = IdListConverter
app.url_map.converters['query'] = QueryConverter

@app.before_request
def before_request():
    g.lib = app.config['lib']

# Items.

@app.route('/item/<idlist:ids>')
@resource('items')
def get_item(id):
    return g.lib.get_item(id)

@app.route('/item/')
@app.route('/item/query/')
@resource_list('items')
def all_items():
    return g.lib.items()

@app.route('/item/<int:item_id>/file')
def item_file(item_id):
    item = g.lib.get_item(item_id)
    response = flask.send_file(
        util.py3_path(item.path),
        as_attachment=True,
        attachment_filename=os.path.basename(util.py3_path(item.path)),
    )
    response.headers['Content-Length'] = os.path.getsize(item.path)
    return response

@app.route('/item/query/<query:queries>')
@resource_query('items')
def item_query(queries):
    return g.lib.items(queries)

@app.route('/item/at_path/<path:path>')
def item_at_path(path):
    try:
        return flask.jsonify(_rep(beets.library.Item.from_path('/' + path)))
    except beets.library.ReadError:
        return flask.abort(404)

@app.route('/item/values/<string:key>')
def item_unique_field_values(key):
    sort_key = flask.request.args.get('sort_key', key)
    try:
        values = _get_unique_table_field_values(beets.library.Item, key,
                                                sort_key)
    except KeyError:
        return flask.abort(404)
    return flask.jsonify(values=values)

# Albums.

@app.route('/album/<idlist:ids>')
@resource('albums')
def get_album(id):
    return g.lib.get_album(id)

@app.route('/album/')
@app.route('/album/query/')
@resource_list('albums')
def all_albums():
    return g.lib.albums()

@app.route('/album/query/<query:queries>')
@resource_query('albums')
def album_query(queries):
    return g.lib.albums(queries)

@app.route('/album/<int:album_id>/art')
def album_art(album_id):
    album = g.lib.get_album(album_id)
    if album.artpath:
        return flask.send_file(album.artpath)
    else:
        return flask.abort(404)

@app.route('/album/values/<string:key>')
def album_unique_field_values(key):
    sort_key = flask.request.args.get('sort_key', key)
    try:
        values = _get_unique_table_field_values(beets.library.Album, key,
                                                sort_key)
    except KeyError:
        return flask.abort(404)
    return flask.jsonify(values=values)

# Artists.

@app.route('/artist/')
def all_artists():
    with g.lib.transaction() as tx:
        rows = tx.query("SELECT DISTINCT albumartist FROM albums")
    all_artists = [row[0] for row in rows]
    return flask.jsonify(artist_names=all_artists)

# Library information.

@app.route('/stats')
def stats():
    with g.lib.transaction() as tx:
        item_rows = tx.query("SELECT COUNT(*) FROM items")
        album_rows = tx.query("SELECT COUNT(*) FROM albums")
    return flask.jsonify({
        'items': item_rows[0][0],
        'albums': album_rows[0][0],
    })

# UI.

@app.route('/')
def home():
    return flask.render_template('index.html')

# Plugin hook.

class WebPlugin(BeetsPlugin):
    def __init__(self):
        super(WebPlugin, self).__init__()
        self.config.add({
            'host': u'127.0.0.1',
            'port': 8337,
            'cors': '',
            'exclude_paths_from_items': True,
        })

    def commands(self):
        cmd = ui.Subcommand('web', help=u'start a Web interface')
        cmd.parser.add_option(u'-d', u'--debug', action='store_true',
                              default=False, help=u'debug mode')

        def func(lib, opts, args):
            args = ui.decargs(args)
            if args:
                self.config['host'] = args.pop(0)
            if args:
                self.config['port'] = int(args.pop(0))

            app.config['lib'] = lib
            # Normalizes json output
            app.config['JSONIFY_PRETTYPRINT_REGULAR'] = False

            app.config['EXCLUDE_PATHS_FROM_ITEMS'] = (
                self.config.get('exclude_paths_from_items', True))

            # Enable CORS if required.
            if self.config['cors']:
                self._log.info(u'Enabling CORS with origin: {0}',
                               self.config['cors'])
                from flask.ext.cors import CORS
                app.config['CORS_ALLOW_HEADERS'] = "Content-Type"
                app.config['CORS_RESOURCES'] = {
                    r"/*": {"origins": self.config['cors'].get(str)}
                }
                CORS(app)
            # Start the web application.
            app.run(host=self.config['host'].as_str(),
                    port=self.config['port'].get(int),
                    debug=opts.debug, threaded=True)
        cmd.func = func
        return [cmd]
<|endoftext|>


<commit_before>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import benchexec.util as util
import benchexec.tools.template
import benchexec.result as result

class Tool(benchexec.tools.template.BaseTool):
    """
    Tool info for Forest
    """
    REQUIRED_PATHS = [
                  "bin",
                  "lib",
                  "tools"
                  ]

    def executable(self):
        return util.find_executable('forest.sh')

    def name(self):
        return 'Forest'

    def version(self, executable):
        return self._version_from_tool(executable)

    def cmdline(self, executable, options, tasks, propertyfile, rlimits):
        assert len(tasks) == 1, "only one inputfile supported"
        return [executable] + ["-propertyfile", propertyfile] + options + ["-svcomp_only_output"] + ["-file"] + [ tasks[0] ]

    def determine_result(self, returncode, returnsignal, output, isTimeout):
        status = result.RESULT_UNKNOWN
        if "TRUE" in output:
            status = result.RESULT_TRUE_PROP
        if "FALSE_REACH" in output:
            status = result.RESULT_FALSE_REACH
        if "FALSE_DEREF" in output:
            status = result.RESULT_FALSE_DEREF
        if "FALSE_FREE" in output:
            status = result.RESULT_FALSE_FREE

        return status
<commit_msg>Correct result translation for Forest<commit_after>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2007-2015  Dirk Beyer
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import benchexec.util as util
import benchexec.tools.template
import benchexec.result as result

class Tool(benchexec.tools.template.BaseTool):
    """
    Tool info for Forest
    """
    REQUIRED_PATHS = [
                  "bin",
                  "lib",
                  "tools"
                  ]

    def executable(self):
        return util.find_executable('forest.sh')

    def name(self):
        return 'Forest'

    def version(self, executable):
        return self._version_from_tool(executable)

    def cmdline(self, executable, options, tasks, propertyfile, rlimits):
        assert len(tasks) == 1, "only one inputfile supported"
        return [executable] + ["-propertyfile", propertyfile] + options + ["-svcomp_only_output"] + ["-file"] + [ tasks[0] ]

    def determine_result(self, returncode, returnsignal, output, isTimeout):
        status = result.RESULT_UNKNOWN
        for line in output:
            if "TRUE" in line:
                status = result.RESULT_TRUE_PROP
            if "FALSE_REACH" in line:
                status = result.RESULT_FALSE_REACH
            if "FALSE_DEREF" in line:
                status = result.RESULT_FALSE_DEREF
            if "FALSE_FREE" in line:
                status = result.RESULT_FALSE_FREE
        return status

<|endoftext|>


<commit_before>
from south.db import db
from django.db import models
from ella.core.models import *

class Migration:

    def forwards(self, orm):

        # Adding model 'Publishable'
        db.create_table('core_publishable', (
            ('id', models.AutoField(primary_key=True)),
            ('content_type', models.ForeignKey(orm['contenttypes.ContentType'])),
            ('category', models.ForeignKey(orm.Category, verbose_name=_('Category'))),
            ('title', models.CharField(_('Title'), max_length=255)),
            ('slug', models.SlugField(_('Slug'), max_length=255)),
            ('source', models.ForeignKey(orm.Source, null=True, verbose_name=_('Source'), blank=True)),
            ('photo', models.ForeignKey(orm['photos.Photo'], null=True, verbose_name=_('Photo'), blank=True)),
            ('description', models.TextField(_('Description'))),
            ('publish_from', models.DateTimeField(_('Publish from'), default=datetime(3000, 1, 1, 0, 0), editable=False)),
        ))
        db.send_create_signal('core', ['Publishable'])

        # Adding model 'Dependency'
        db.create_table('core_dependency', (
            ('id', models.AutoField(primary_key=True)),
            ('target_ct', models.ForeignKey(orm['contenttypes.ContentType'], related_name='dependency_for_set')),
            ('target_id', models.IntegerField()),
            ('dependent_ct', models.ForeignKey(orm['contenttypes.ContentType'], related_name='depends_on_set')),
            ('dependent_id', models.IntegerField()),
        ))
        db.send_create_signal('core', ['Dependency'])

        # Adding ManyToManyField 'Publishable.authors'
        db.create_table('core_publishable_authors', (
            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
            ('publishable', models.ForeignKey(orm.Publishable, null=False)),
            ('author', models.ForeignKey(orm.Author, null=False))
        ))

    def backwards(self, orm):

        # Dropping ManyToManyField 'Publishable.authors'
        db.delete_table('core_publishable_authors')

        # Deleting model 'Dependency'
        db.delete_table('core_dependency')

        # Deleting model 'Publishable'
        db.delete_table('core_publishable')

    models = {
    }

<commit_msg>trying to add all frozen orms<commit_after>
from south.db import db
from django.db import models
from ella.core.models import *

class Migration:

    def forwards(self, orm):

        # Adding model 'Publishable'
        db.create_table('core_publishable', (
            ('id', models.AutoField(primary_key=True)),
            ('content_type', models.ForeignKey(orm['contenttypes.ContentType'])),
            ('category', models.ForeignKey(orm.Category, verbose_name=_('Category'))),
            ('title', models.CharField(_('Title'), max_length=255)),
            ('slug', models.SlugField(_('Slug'), max_length=255)),
            ('source', models.ForeignKey(orm.Source, null=True, verbose_name=_('Source'), blank=True)),
            ('photo', models.ForeignKey(orm['photos.Photo'], null=True, verbose_name=_('Photo'), blank=True)),
            ('description', models.TextField(_('Description'))),
            ('publish_from', models.DateTimeField(_('Publish from'), default=datetime(3000, 1, 1, 0, 0), editable=False)),
        ))
        db.send_create_signal('core', ['Publishable'])

        # Adding model 'Dependency'
        db.create_table('core_dependency', (
            ('id', models.AutoField(primary_key=True)),
            ('target_ct', models.ForeignKey(orm['contenttypes.ContentType'], related_name='dependency_for_set')),
            ('target_id', models.IntegerField()),
            ('dependent_ct', models.ForeignKey(orm['contenttypes.ContentType'], related_name='depends_on_set')),
            ('dependent_id', models.IntegerField()),
        ))
        db.send_create_signal('core', ['Dependency'])

        # Adding ManyToManyField 'Publishable.authors'
        db.create_table('core_publishable_authors', (
            ('id', models.AutoField(verbose_name='ID', primary_key=True, auto_created=True)),
            ('publishable', models.ForeignKey(orm.Publishable, null=False)),
            ('author', models.ForeignKey(orm.Author, null=False))
        ))

    def backwards(self, orm):

        # Dropping ManyToManyField 'Publishable.authors'
        db.delete_table('core_publishable_authors')

        # Deleting model 'Dependency'
        db.delete_table('core_dependency')

        # Deleting model 'Publishable'
        db.delete_table('core_publishable')

    models = {
        # orm.Category
        'core.category': {
            'Meta': {'unique_together': "(('site','tree_path'),)", 'app_label': "'core'"},
            'description': ('models.TextField', ['_("Category Description")'], {'blank': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'site': ('models.ForeignKey', ["orm['sites.Site']"], {}),
            'slug': ('models.SlugField', ["_('Slug')"], {'max_length': '255'}),
            'title': ('models.CharField', ['_("Category Title")'], {'max_length': '200'}),
            'tree_parent': ('models.ForeignKey', ["orm['core.Category']"], {'null': 'True', 'verbose_name': '_("Parent Category")', 'blank': 'True'}),
            'tree_path': ('models.CharField', [], {'editable': 'False', 'max_length': '255', 'verbose_name': '_("Path from root category")'})
        },
        'core.placement': {
            'Meta': {'app_label': "'core'"},
            'category': ('models.ForeignKey', ["orm['core.Category']"], {'verbose_name': "_('Category')", 'db_index': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'publish_from': ('models.DateTimeField', ['_("Start of visibility")'], {}),
            'publish_to': ('models.DateTimeField', ['_("End of visibility")'], {'null': 'True', 'blank': 'True'}),
            'publishable': ('models.ForeignKey', ["orm['core.Publishable']"], {'verbose_name': "_('Publishable object')"}),
            'slug': ('models.SlugField', ["_('Slug')"], {'max_length': '255', 'blank': 'True'}),
            'static': ('models.BooleanField', ["_('static')"], {'default': 'False'})
        },
        'core.listing': {
            'Meta': {'ordering': "('-publish_from',)", 'app_label': "'core'"},
            'category': ('models.ForeignKey', ["orm['core.Category']"], {'verbose_name': "_('Category')", 'db_index': 'True'}),
            'commercial': ('models.BooleanField', ['_("Commercial")'], {'default': 'False'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'placement': ('models.ForeignKey', ["orm['core.Placement']"], {'verbose_name': "_('Placement')"}),
            'priority_from': ('models.DateTimeField', ['_("Start of prioritized listing")'], {'null': 'True', 'blank': 'True'}),
            'priority_to': ('models.DateTimeField', ['_("End of prioritized listing")'], {'null': 'True', 'blank': 'True'}),
            'priority_value': ('models.IntegerField', ['_("Priority")'], {'null': 'True', 'blank': 'True'}),
            'publish_from': ('models.DateTimeField', ['_("Start of listing")'], {}),
            'publish_to': ('models.DateTimeField', ['_("End of listing")'], {'null': 'True', 'blank': 'True'})
        },
        'core.hitcount': {
            'Meta': {'app_label': "'core'"},
            'hits': ('models.PositiveIntegerField', ["_('Hits')"], {'default': '1'}),
            'last_seen': ('models.DateTimeField', ["_('Last seen')"], {'editable': 'False'}),
            'placement': ('models.ForeignKey', ["orm['core.Placement']"], {'primary_key': 'True'})
        },
        'core.related': {
            'Meta': {'app_label': "'core'"},
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'publishable': ('models.ForeignKey', ["orm['core.Publishable']"], {'verbose_name': "_('Publishable')"}),
            'related_ct': ('models.ForeignKey', ["orm['contenttypes.ContentType']"], {'verbose_name': "_('Content type')"}),
            'related_id': ('models.IntegerField', ["_('Object ID')"], {})
        },
        'sites.site': {
            'Meta': {'ordering': "('domain',)", 'db_table': "'django_site'"},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        'auth.user': {
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        # orm['photos.Photo']
        'photos.photo': {
            'Meta': {'ordering': "('-created',)"},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        # orm.Publishable
        'core.publishable': {
            'Meta': {'app_label': "'core'"},
            'authors': ('models.ManyToManyField', ["orm['core.Author']"], {'verbose_name': "_('Authors')"}),
            'category': ('models.ForeignKey', ["orm['core.Category']"], {'verbose_name': "_('Category')"}),
            'content_type': ('models.ForeignKey', ["orm['contenttypes.ContentType']"], {}),
            'description': ('models.TextField', ["_('Description')"], {}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'photo': ('models.ForeignKey', ["orm['photos.Photo']"], {'null': 'True', 'verbose_name': "_('Photo')", 'blank': 'True'}),
            'publish_from': ('models.DateTimeField', ["_('Publish from')"], {'default': 'datetime(3000, 1, 1, 0, 0)', 'editable': 'False'}),
            'slug': ('models.SlugField', ["_('Slug')"], {'max_length': '255'}),
            'source': ('models.ForeignKey', ["orm['core.Source']"], {'null': 'True', 'verbose_name': "_('Source')", 'blank': 'True'}),
            'title': ('models.CharField', ["_('Title')"], {'max_length': '255'})
        },
        # orm['contenttypes.ContentType']
        'contenttypes.contenttype': {
            'Meta': {'ordering': "('name',)", 'unique_together': "(('app_label','model'),)", 'db_table': "'django_content_type'"},
            '_stub': True,
            'id': ('models.AutoField', [], {'primary_key': 'True'})
        },
        # orm.Source
        'core.source': {
            'Meta': {'app_label': "'core'"},
            'description': ('models.TextField', ["_('Description')"], {'blank': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', ["_('Name')"], {'max_length': '200'}),
            'url': ('models.URLField', ["_('URL')"], {'blank': 'True'})
        },
        'core.dependency': {
            'Meta': {'ordering': "('dependent_ct','dependent_id',)", 'app_label': "'core'"},
            'dependent_ct': ('models.ForeignKey', ["orm['contenttypes.ContentType']"], {'related_name': "'depends_on_set'"}),
            'dependent_id': ('models.IntegerField', [], {}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'target_ct': ('models.ForeignKey', ["orm['contenttypes.ContentType']"], {'related_name': "'dependency_for_set'"}),
            'target_id': ('models.IntegerField', [], {})
        },
        # orm.Author
        'core.author': {
            'Meta': {'app_label': "'core'"},
            'description': ('models.TextField', ["_('Description')"], {'blank': 'True'}),
            'email': ('models.EmailField', ["_('Email')"], {'blank': 'True'}),
            'id': ('models.AutoField', [], {'primary_key': 'True'}),
            'name': ('models.CharField', ["_('Name')"], {'max_length': '200', 'blank': 'True'}),
            'slug': ('models.SlugField', ["_('Slug')"], {'unique': 'True', 'max_length': '255'}),
            'text': ('models.TextField', ["_('Text')"], {'blank': 'True'}),
            'user': ('models.ForeignKey', ["orm['auth.User']"], {'null': 'True', 'verbose_name': "_('User')", 'blank': 'True'})
        }
    }

<|endoftext|>


<commit_before>import os

import pandas as pd
import pytest
from pandas.util import testing as pdt
from numpy.testing import assert_almost_equal
from tardis.io.config_reader import Configuration
from tardis.simulation import Simulation

class BasePlasmaTest(object):
    #Class defining all common tests for different setups of Plasma
    #This can then be inherited for different Plasma setup

    @pytest.fixture(scope="class")
    def plasma(self, atomic_data_fname, tardis_ref_path, data):
        config = data['config']
        config['atom_data'] = atomic_data_fname
        sim = Simulation.from_config(config)

        if pytest.config.getvalue("--generate-reference"):
            if os.path.exists(data['reference_file_path']):
                pytest.skip(
                    'Reference data {0} does exist and tests will not '
                    'proceed generating new data'.format(data['reference_file_path']))
            sim.plasma.to_hdf(data['reference_file_path'])
            pytest.skip("Reference data saved at {0}".format(
                data['reference_file_path']))
        return sim.plasma

    @pytest.fixture(scope="class")
    def data(self):
        pass

    @pytest.yield_fixture()
    def reference(self, data):
        with pd.HDFStore(data['reference_file_path']) as hdf_file:
            yield hdf_file

    general_properties = ['beta_rad', 'g_electron', 'selected_atoms',
                          'number_density', 't_electrons', 'w', 't_rad']
    partiton_properties = ['level_boltzmann_factor', 'partition_function']
    atomic_properties = ['excitation_energy', 'lines', 'lines_lower_level_index',
                         'lines_upper_level_index', 'atomic_mass', 'ionization_data',
                         'nu', 'wavelength_cm', 'f_lu', 'metastability']
    ion_population_properties = [
        'phi', 'ion_number_density', 'electron_densities']
    level_population_properties = ['level_number_density']
    radiative_properties = ['stimulated_emission_factor',
                            'tau_sobolevs', 'beta_sobolev', 'transition_probabilities']
    j_blues_properties = ['j_blues']

    combined_properties = general_properties + partiton_properties + atomic_properties + ion_population_properties + \
        level_population_properties + radiative_properties + \
        j_blues_properties

    @pytest.mark.parametrize("attr", combined_properties)
    def test_plasma_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if actual.ndim == 1:
            actual = pd.Series(actual)
        else:
            actual = pd.DataFrame(actual)
        expected = reference.select(os.path.join('plasma', attr))
        pdt.assert_almost_equal(actual, expected)

    def test_levels(self, plasma, reference):
        actual = pd.DataFrame(plasma.levels)
        expected = reference.select(os.path.join('plasma', 'levels'))
        pdt.assert_almost_equal(actual, expected)

    scalars_properties = ['time_explosion', 'link_t_rad_t_electron']

    @pytest.mark.parametrize("attr", scalars_properties)
    def test_scalars_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if hasattr(actual, 'cgs'):
            actual = actual.cgs.value
        expected = reference.select(os.path.join(
            'plasma', 'scalars'))[attr]
        pdt.assert_almost_equal(actual, expected)

    def test_helium_treatment(self, plasma, reference):
        actual = plasma.helium_treatment
        expected = reference.select(os.path.join(
            'plasma', 'scalars'))['helium_treatment']
        assert actual == expected

class TestLTEPlasma(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_lte_reference.h5')
        config_path = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_lte.yml')
        plasma_data['config'] = Configuration.from_yaml(config_path)
        return plasma_data

class TestNLTEPlasma(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_nlte_reference.h5')
        config_path = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_nlte.yml')
        plasma_data['config'] = Configuration.from_yaml(config_path)
        return plasma_data

    #Additional Tests for NLTE Plasma

    nlte_general_properties = ['beta_electron']
    nlte_ion_population_properties = ['delta', 'previous_electron_densities']
    nlte_radiative_properties = ['previous_beta_sobolev']
    nlte_properties = nlte_general_properties + \
        nlte_ion_population_properties + nlte_radiative_properties

    @pytest.mark.parametrize("attr", nlte_properties)
    def test_nlte_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if actual.ndim == 1:
            actual = pd.Series(actual)
        else:
            actual = pd.DataFrame(actual)
        expected = reference.select(os.path.join('plasma', attr))
        pdt.assert_almost_equal(actual, expected)

    def test_zeta_data(self, plasma, reference):
        actual = getattr(plasma, 'zeta_data')
        expected = reference.select(os.path.join('plasma', 'zeta_data'))
        assert_almost_equal(actual, expected.values)
<commit_msg>Add some more plasma tests<commit_after>import os

import pandas as pd
import pytest
from pandas.util import testing as pdt
from numpy.testing import assert_almost_equal
from tardis.io.config_reader import Configuration
from tardis.simulation import Simulation

class BasePlasmaTest(object):
    #Class defining all common tests for different setups of Plasma
    #This can then be inherited for different Plasma setup

    @pytest.fixture(scope="class")
    def plasma(self, atomic_data_fname, tardis_ref_path, data):
        config = data['config']
        config['atom_data'] = atomic_data_fname
        sim = Simulation.from_config(config)

        if pytest.config.getvalue("--generate-reference"):
            if os.path.exists(data['reference_file_path']):
                pytest.skip(
                    'Reference data {0} does exist and tests will not '
                    'proceed generating new data'.format(data['reference_file_path']))
            sim.plasma.to_hdf(data['reference_file_path'])
            pytest.skip("Reference data saved at {0}".format(
                data['reference_file_path']))
        return sim.plasma

    @pytest.fixture(scope="class")
    def data(self):
        pass

    @pytest.yield_fixture()
    def reference(self, data):
        with pd.HDFStore(data['reference_file_path']) as hdf_file:
            yield hdf_file

    general_properties = ['beta_rad', 'g_electron', 'selected_atoms',
                          'number_density', 't_electrons', 'w', 't_rad']
    partiton_properties = ['level_boltzmann_factor', 'partition_function']
    atomic_properties = ['excitation_energy', 'lines', 'lines_lower_level_index',
                         'lines_upper_level_index', 'atomic_mass', 'ionization_data',
                         'nu', 'wavelength_cm', 'f_lu', 'metastability']
    ion_population_properties = [
        'phi', 'ion_number_density', 'electron_densities']
    level_population_properties = ['level_number_density']
    radiative_properties = ['stimulated_emission_factor',
                            'tau_sobolevs', 'beta_sobolev', 'transition_probabilities']
    j_blues_properties = ['j_blues']

    combined_properties = general_properties + partiton_properties + atomic_properties + ion_population_properties + \
        level_population_properties + radiative_properties + \
        j_blues_properties

    @pytest.mark.parametrize("attr", combined_properties)
    def test_plasma_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if actual.ndim == 1:
            actual = pd.Series(actual)
        else:
            actual = pd.DataFrame(actual)
        expected = reference.select(os.path.join('plasma', attr))
        pdt.assert_almost_equal(actual, expected)

    def test_levels(self, plasma, reference):
        actual = pd.DataFrame(plasma.levels)
        expected = reference.select(os.path.join('plasma', 'levels'))
        pdt.assert_almost_equal(actual, expected)

    scalars_properties = ['time_explosion', 'link_t_rad_t_electron']

    @pytest.mark.parametrize("attr", scalars_properties)
    def test_scalars_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if hasattr(actual, 'cgs'):
            actual = actual.cgs.value
        expected = reference.select(os.path.join(
            'plasma', 'scalars'))[attr]
        pdt.assert_almost_equal(actual, expected)

    def test_helium_treatment(self, plasma, reference):
        actual = plasma.helium_treatment
        expected = reference.select(os.path.join(
            'plasma', 'scalars'))['helium_treatment']
        assert actual == expected

class TestLTEPlasma(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_lte_reference.h5')
        config_path = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_lte.yml')
        plasma_data['config'] = Configuration.from_yaml(config_path)
        return plasma_data

class TestNLTEPlasma(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_nlte_reference.h5')
        config_path = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_nlte.yml')
        plasma_data['config'] = Configuration.from_yaml(config_path)
        return plasma_data

    #Additional Tests for NLTE Plasma

    nlte_general_properties = ['beta_electron']
    nlte_ion_population_properties = ['delta', 'previous_electron_densities']
    nlte_radiative_properties = ['previous_beta_sobolev']
    nlte_properties = nlte_general_properties + \
        nlte_ion_population_properties + nlte_radiative_properties

    @pytest.mark.parametrize("attr", nlte_properties)
    def test_nlte_properties(self, plasma, reference, attr):
        actual = getattr(plasma, attr)
        if actual.ndim == 1:
            actual = pd.Series(actual)
        else:
            actual = pd.DataFrame(actual)
        expected = reference.select(os.path.join('plasma', attr))
        pdt.assert_almost_equal(actual, expected)

    def test_zeta_data(self, plasma, reference):
        actual = getattr(plasma, 'zeta_data')
        expected = reference.select(os.path.join('plasma', 'zeta_data'))
        assert_almost_equal(actual, expected.values)

class TestPlasmaSetupIII(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_setup_III_reference.h5')
        plasma_data['config_path'] = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_nlte.yml')
        config = Configuration.from_yaml(plasma_data['config_path'])
        config.plasma.radiative_rates_type = 'detailed'
        config.plasma.nlte.classical_nebular = True
        #config.plasma.helium_treatment = 'recomb-nlte'
        plasma_data['config'] = config
        return plasma_data

    j_blues_detailed_properties = ['j_blues_norm_factor', 'j_blue_estimator']
    additional_properties = j_blues_detailed_properties + ['volume', 'r_inner']

    @pytest.mark.parametrize("attr", additional_properties)
    def test_j_blues_detailed_properties(self, plasma, reference,  attr):
        actual = getattr(plasma, attr)
        if actual.ndim == 1:
            actual = pd.Series(actual)
        else:
            actual = pd.DataFrame(actual)
        expected = reference.select(os.path.join('plasma', attr))
        pdt.assert_almost_equal(actual, expected)

class TestPlasmaSetupIV(BasePlasmaTest):

    @pytest.fixture(scope="class")
    def data(self, tardis_ref_path):
        plasma_data = {}
        plasma_data['reference_file_path'] = os.path.join(
            tardis_ref_path, 'plasma_reference', 'plasma_setup_IV_reference.h5')
        plasma_data['config_path'] = os.path.join(
            'tardis', 'plasma', 'tests', 'data', 'plasma_test_config_nlte.yml')
        config = Configuration.from_yaml(plasma_data['config_path'])
        config.plasma.radiative_rates_type = 'blackbody'
        #config.plasma.nlte.coronal_approximation = True
        #config.plasma.helium_treatment = 'numerical-nlte'
        plasma_data['config'] = config
        return plasma_data
<|endoftext|>


<commit_before>"""
Invoker that uses multiple processes.
"""
from __future__ import division, print_function, with_statement

import uuid
from time import sleep
from threading import Lock
from multiprocessing import Manager

from orges.invoker.base import BaseInvoker
from orges.util.stoppable import stopping_method, stoppable_method
from orges.invoker.util.task_handle import TaskHandle
from orges.invoker.util.determine_package import determine_package
from orges.invoker.util.determine_worker_count import determine_worker_count
from orges.invoker.util.worker_provider import WorkerProcessProvider
from orges.invoker.util.model import Result, Error, Task

class MultiProcessInvoker(BaseInvoker):
    """
    Invoker that invokes objective functions in parallel using processes.
    """

    def __init__(self, resources=None):
        """
        :param  resources: Number of CPUs to use at most. Will automatically
                           configure itself, if None.
        """

        self._worker_count_max = determine_worker_count(resources)

        # set this using the property
        self._caller = None

        # queues common to all worker processes
        manager = Manager()
        self._queue_results = manager.Queue()
        self._queue_status = manager.Queue()
        self._queue_tasks = manager.Queue()

        self._worker_handles = []
        self._worker_provider = WorkerProcessProvider()

        # initialize logging
        #self._logger = multiprocessing.log_to_stderr(logging.INFO)

        # we can not prohibit others to use us in parallel, so
        # make this invoker thread-safe
        self._lock = Lock()

        super(MultiProcessInvoker, self).__init__(self.caller)

    @property
    def caller(self):
        """Property for the private caller attribute."""
        with self._lock:
            return self._caller

    @caller.setter
    def caller(self, value):
        """Setter for the private caller attribute."""
        with self._lock:
            self._caller = value

    @stoppable_method
    def invoke(self, function, fargs, *vargs, **kwargs):
        """
        Invokes call(f, fargs) with the given function and the given arguments.

        Calls back to self.caller.on_result() for successful invokes.
        Calls back to self.caller.on_error() for unsuccessful invokes.
        Can be called asynchronously, but will block if the call can not be
        executed immediately, especially when using multiple processes/threads.
        """
        with self._lock:
            # try to provision one new worker
            try:
                self._worker_handles.append(self._worker_provider.provision(
                    queue_tasks=self._queue_tasks,
                    queue_results=self._queue_results,
                    queue_status=self._queue_status))
            except IndexError:
                pass  # give the task to one of the busy workers

            # schedule task for any worker to execute
            self._queue_tasks.put(Task(task_id=uuid.uuid4(),
                                       function=determine_package(function),
                                       args=fargs, kwargs=kwargs))

            # wait for any worker to start working on the task
            status = self._queue_status.get()

            if status is None:
                # TODO Why should status be None?
                task_handle = None
            else:
                task_handle = TaskHandle(invoker=self, task_id=status.task_id)

            # if at worker limit, wait for an result
            if len(self._worker_handles) is self._worker_count_max:
                result = self._queue_results.get()

                # handle finished worker
                if result is None:
                    for worker_handle in self._worker_handles:
                        if (
                                worker_handle.worker_id == result.worker_id
                                and worker_handle.current_task_id ==
                                result.task_id):
                            worker_handle.stop()
                            self._worker_handles.remove(worker_handle)

                # handle regular results
                if isinstance(result, Result):
                    self._caller.on_result(result=result.value,
                                           fargs=result.args,
                                           vargs=result.vargs,
                                           **result.kwargs)
                # handle error results
                if isinstance(result, Error):
                    self._caller.on_error(result=result.value,
                                          fargs=result.args,
                                          **result.kwargs)

            return task_handle

    @stoppable_method
    @stopping_method
    def stop(self):
        """Terminates all worker processes for immediate shutdown."""
        with self._lock:
            for worker_handle in self._worker_handles:
                worker_handle.stop()
                self._worker_handles.remove(worker_handle)
            return

    def stop_task(self, task_id):
        """Terminates a worker_handle given by id."""
        with self._lock:
            for worker_handle in self._worker_handles:
                if worker_handle.current_task_id != task_id:
                    continue
                worker_handle.stop()
                self._worker_handles.remove(worker_handle)
                return  # worker handle is unique, don't look any further

    def wait(self):
        """Blocks until all tasks are done."""
        busy = True
        while busy:
            busy = False
            for handle in self._worker_handles:
                if handle.busy:
                    busy = True
                    sleep(0.1)
                    break

    def get_subinvoker(self, resources):
        """Returns a subinvoker using the given amount of resources of self."""
        del resources
        raise NotImplementedError()
<commit_msg>remove vargs in MPI<commit_after>"""
Invoker that uses multiple processes.
"""
from __future__ import division, print_function, with_statement

import uuid
from time import sleep
from threading import Lock
from multiprocessing import Manager

from orges.invoker.base import BaseInvoker
from orges.util.stoppable import stopping_method, stoppable_method
from orges.invoker.util.task_handle import TaskHandle
from orges.invoker.util.determine_package import determine_package
from orges.invoker.util.determine_worker_count import determine_worker_count
from orges.invoker.util.worker_provider import WorkerProcessProvider
from orges.invoker.util.model import Result, Error, Task

class MultiProcessInvoker(BaseInvoker):
    """
    Invoker that invokes objective functions in parallel using processes.
    """

    def __init__(self, resources=None):
        """
        :param  resources: Number of CPUs to use at most. Will automatically
                           configure itself, if None.
        """

        self._worker_count_max = determine_worker_count(resources)

        # set this using the property
        self._caller = None

        # queues common to all worker processes
        manager = Manager()
        self._queue_results = manager.Queue()
        self._queue_status = manager.Queue()
        self._queue_tasks = manager.Queue()

        self._worker_handles = []
        self._worker_provider = WorkerProcessProvider()

        # initialize logging
        #self._logger = multiprocessing.log_to_stderr(logging.INFO)

        # we can not prohibit others to use us in parallel, so
        # make this invoker thread-safe
        self._lock = Lock()

        super(MultiProcessInvoker, self).__init__(self.caller)

    @property
    def caller(self):
        """Property for the private caller attribute."""
        with self._lock:
            return self._caller

    @caller.setter
    def caller(self, value):
        """Setter for the private caller attribute."""
        with self._lock:
            self._caller = value

    @stoppable_method
    def invoke(self, function, fargs, *vargs, **kwargs):
        """
        Invokes call(f, fargs) with the given function and the given arguments.

        Calls back to self.caller.on_result() for successful invokes.
        Calls back to self.caller.on_error() for unsuccessful invokes.
        Can be called asynchronously, but will block if the call can not be
        executed immediately, especially when using multiple processes/threads.
        """
        with self._lock:
            # try to provision one new worker
            try:
                self._worker_handles.append(self._worker_provider.provision(
                    queue_tasks=self._queue_tasks,
                    queue_results=self._queue_results,
                    queue_status=self._queue_status))
            except IndexError:
                pass  # give the task to one of the busy workers

            # schedule task for any worker to execute
            self._queue_tasks.put(Task(task_id=uuid.uuid4(),
                                       function=determine_package(function),
                                       args=fargs, kwargs=kwargs))

            # wait for any worker to start working on the task
            status = self._queue_status.get()

            if status is None:
                # TODO Why should status be None?
                task_handle = None
            else:
                task_handle = TaskHandle(invoker=self, task_id=status.task_id)

            # if at worker limit, wait for an result
            if len(self._worker_handles) is self._worker_count_max:
                result = self._queue_results.get()

                # handle finished worker
                if result is None:
                    for worker_handle in self._worker_handles:
                        if (
                                worker_handle.worker_id == result.worker_id
                                and worker_handle.current_task_id ==
                                result.task_id):
                            worker_handle.stop()
                            self._worker_handles.remove(worker_handle)

                # handle regular results
                if isinstance(result, Result):
                    self._caller.on_result(result=result.value,
                                           fargs=result.args,
                                           **result.kwargs)
                # handle error results
                if isinstance(result, Error):
                    self._caller.on_error(result=result.value,
                                          fargs=result.args,
                                          **result.kwargs)

            return task_handle

    @stoppable_method
    @stopping_method
    def stop(self):
        """Terminates all worker processes for immediate shutdown."""
        with self._lock:
            for worker_handle in self._worker_handles:
                worker_handle.stop()
                self._worker_handles.remove(worker_handle)
            return

    def stop_task(self, task_id):
        """Terminates a worker_handle given by id."""
        with self._lock:
            for worker_handle in self._worker_handles:
                if worker_handle.current_task_id != task_id:
                    continue
                worker_handle.stop()
                self._worker_handles.remove(worker_handle)
                return  # worker handle is unique, don't look any further

    def wait(self):
        """Blocks until all tasks are done."""
        busy = True
        while busy:
            busy = False
            for handle in self._worker_handles:
                if handle.busy:
                    busy = True
                    sleep(0.1)
                    break

    def get_subinvoker(self, resources):
        """Returns a subinvoker using the given amount of resources of self."""
        del resources
        raise NotImplementedError()
<|endoftext|>


<commit_before>import os
import pwd
import shutil
import socket
import stat
import unittest

import osgtest.library.core as core
import osgtest.library.files as files
import osgtest.library.tomcat as tomcat

class TestStartGUMS(unittest.TestCase):

    # ==========================================================================
    # START: COPIED FROM test_20_voms.py
    # Should be refactored!!!
    # ==========================================================================

    def check_file_and_perms(self, file_path, owner_name, permissions):
        """Return True if the file at 'file_path' exists, is owned by
        'owner_name', is a file, and has the given permissions; False otherwise

        """
        owner_uid = pwd.getpwnam(owner_name)
        try:
            file_stat = os.stat(file_path)
            return (file_stat.st_uid == owner_uid and
                    file_stat.st_mode & 07777 == permissions and
                    stat.S_ISREG(file_stat.st_mode))
        except OSError: # file does not exist
            return False

    # Carefully install a certificate with the given key from the given
    # source path, then set ownership and permissions as given.  Record
    # each directory and file created by this process into the config
    # dictionary; do so immediately after creation, so that the
    # remove_cert() function knows exactly what to remove/restore.
    def install_cert(self, target_key, source_key, owner_name, permissions):
        target_path = core.config[target_key]
        target_dir = os.path.dirname(target_path)
        source_path = core.config[source_key]
        user = pwd.getpwnam(owner_name)

        # Using os.path.lexists because os.path.exists return False for broken symlinks
        if os.path.lexists(target_path):
            backup_path = target_path + '.osgtest.backup'
            shutil.move(target_path, backup_path)
            core.state[target_key + '-backup'] = backup_path

        if not os.path.exists(target_dir):
            os.makedirs(target_dir)
            core.state[target_key + '-dir'] = target_dir
            os.chown(target_dir, user.pw_uid, user.pw_gid)
            os.chmod(target_dir, 0755)

        shutil.copy(source_path, target_path)
        core.state[target_key] = target_path
        os.chown(target_path, user.pw_uid, user.pw_gid)
        os.chmod(target_path, permissions)

    def test_01_config_certs(self):
        core.config['certs.hostcert'] = '/etc/grid-security/hostcert.pem'
        core.config['certs.hostkey'] = '/etc/grid-security/hostkey.pem'
        core.config['certs.httpcert'] = '/etc/grid-security/http/httpcert.pem'
        core.config['certs.httpkey'] = '/etc/grid-security/http/httpkey.pem'

    def test_02_install_http_certs(self):
        if core.missing_rpm('gums-service'):
            return
        httpcert = core.config['certs.httpcert']
        httpkey = core.config['certs.httpkey']
        if self.check_file_and_perms(httpcert, 'tomcat', 0644) and self.check_file_and_perms(httpkey, 'tomcat', 0400):
            core.skip('HTTP cert exists and has proper permissions')
            return
        self.install_cert('certs.httpcert', 'certs.hostcert', 'tomcat', 0644)
        self.install_cert('certs.httpkey', 'certs.hostkey', 'tomcat', 0400)

    def test_03_gums_configuration(self):
        core.config['gums.password'] = 'osgGUMS!'

    def test_04_setup_gums_database(self):
        if core.missing_rpm('gums-service'):
            return
        command = ('gums-setup-mysql-database', '--noprompt', '--user', 'gums', '--host', 'localhost:3306',
                   '--password', core.config['gums.password'])
        core.check_system(command, 'Set up GUMS MySQL database')

    def test_05_add_mysql_admin(self):
        if core.missing_rpm('gums-service'):
            return
        host_dn, host_issuer = core.certificate_info(core.config['certs.hostcert'])
        mysql_template_path = '/usr/lib/gums/sql/addAdmin.mysql'
        self.assert_(os.path.exists(mysql_template_path), 'GUMS MySQL template exists')
        mysql_template = files.read(mysql_template_path, as_single_string=True).strip()
        core.log_message(mysql_template)

        mysql_command = re.sub(r'@ADMINDN@', host_dn, mysql_template)
        core.log_message(mysql_command)

        command = ('mysql', '--user=gums', '-p' + core.config['gums.password'], '--execute=' + mysql_command)
        core.check_system(command, 'Add GUMS MySQL admin')

    def test_06_write_gums_config(self):
        if core.missing_rpm('gums-service'):
            return

        # Debugging -- can be deleted later
        command = ('cat', '/etc/gums/gums.config')
        core.check_system(command, 'Dump gums.config')
<commit_msg>Add re import.<commit_after>import os
import pwd
import re
import shutil
import socket
import stat
import unittest

import osgtest.library.core as core
import osgtest.library.files as files
import osgtest.library.tomcat as tomcat

class TestStartGUMS(unittest.TestCase):

    # ==========================================================================
    # START: COPIED FROM test_20_voms.py
    # Should be refactored!!!
    # ==========================================================================

    def check_file_and_perms(self, file_path, owner_name, permissions):
        """Return True if the file at 'file_path' exists, is owned by
        'owner_name', is a file, and has the given permissions; False otherwise

        """
        owner_uid = pwd.getpwnam(owner_name)
        try:
            file_stat = os.stat(file_path)
            return (file_stat.st_uid == owner_uid and
                    file_stat.st_mode & 07777 == permissions and
                    stat.S_ISREG(file_stat.st_mode))
        except OSError: # file does not exist
            return False

    # Carefully install a certificate with the given key from the given
    # source path, then set ownership and permissions as given.  Record
    # each directory and file created by this process into the config
    # dictionary; do so immediately after creation, so that the
    # remove_cert() function knows exactly what to remove/restore.
    def install_cert(self, target_key, source_key, owner_name, permissions):
        target_path = core.config[target_key]
        target_dir = os.path.dirname(target_path)
        source_path = core.config[source_key]
        user = pwd.getpwnam(owner_name)

        # Using os.path.lexists because os.path.exists return False for broken symlinks
        if os.path.lexists(target_path):
            backup_path = target_path + '.osgtest.backup'
            shutil.move(target_path, backup_path)
            core.state[target_key + '-backup'] = backup_path

        if not os.path.exists(target_dir):
            os.makedirs(target_dir)
            core.state[target_key + '-dir'] = target_dir
            os.chown(target_dir, user.pw_uid, user.pw_gid)
            os.chmod(target_dir, 0755)

        shutil.copy(source_path, target_path)
        core.state[target_key] = target_path
        os.chown(target_path, user.pw_uid, user.pw_gid)
        os.chmod(target_path, permissions)

    def test_01_config_certs(self):
        core.config['certs.hostcert'] = '/etc/grid-security/hostcert.pem'
        core.config['certs.hostkey'] = '/etc/grid-security/hostkey.pem'
        core.config['certs.httpcert'] = '/etc/grid-security/http/httpcert.pem'
        core.config['certs.httpkey'] = '/etc/grid-security/http/httpkey.pem'

    def test_02_install_http_certs(self):
        if core.missing_rpm('gums-service'):
            return
        httpcert = core.config['certs.httpcert']
        httpkey = core.config['certs.httpkey']
        if self.check_file_and_perms(httpcert, 'tomcat', 0644) and self.check_file_and_perms(httpkey, 'tomcat', 0400):
            core.skip('HTTP cert exists and has proper permissions')
            return
        self.install_cert('certs.httpcert', 'certs.hostcert', 'tomcat', 0644)
        self.install_cert('certs.httpkey', 'certs.hostkey', 'tomcat', 0400)

    def test_03_gums_configuration(self):
        core.config['gums.password'] = 'osgGUMS!'

    def test_04_setup_gums_database(self):
        if core.missing_rpm('gums-service'):
            return
        command = ('gums-setup-mysql-database', '--noprompt', '--user', 'gums', '--host', 'localhost:3306',
                   '--password', core.config['gums.password'])
        core.check_system(command, 'Set up GUMS MySQL database')

    def test_05_add_mysql_admin(self):
        if core.missing_rpm('gums-service'):
            return
        host_dn, host_issuer = core.certificate_info(core.config['certs.hostcert'])
        mysql_template_path = '/usr/lib/gums/sql/addAdmin.mysql'
        self.assert_(os.path.exists(mysql_template_path), 'GUMS MySQL template exists')
        mysql_template = files.read(mysql_template_path, as_single_string=True).strip()
        core.log_message(mysql_template)

        mysql_command = re.sub(r'@ADMINDN@', host_dn, mysql_template)
        core.log_message(mysql_command)

        command = ('mysql', '--user=gums', '-p' + core.config['gums.password'], '--execute=' + mysql_command)
        core.check_system(command, 'Add GUMS MySQL admin')

    def test_06_write_gums_config(self):
        if core.missing_rpm('gums-service'):
            return

        # Debugging -- can be deleted later
        command = ('cat', '/etc/gums/gums.config')
        core.check_system(command, 'Dump gums.config')
<|endoftext|>