<commit_before># -*- coding: utf-8 -*-
#
# This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
import os
import cStringIO
import copy

from lxml import etree
from librarian import XHTMLNS, ParseError, OutputFile
from librarian import functions

from lxml.etree import XMLSyntaxError, XSLTApplyError

functions.reg_substitute_entities()
functions.reg_person_name()

STYLESHEETS = {
    'legacy': 'xslt/book2html.xslt',
    'full': 'xslt/wl2html_full.xslt',
    'partial': 'xslt/wl2html_partial.xslt'
}

def get_stylesheet(name):
    return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])

def html_has_content(text):
    return etree.ETXPath('//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)})(text)

def transform(wldoc, stylesheet='legacy', options=None, flags=None):
    """Transforms the WL document to XHTML.

    If output_filename is None, returns an XML,
    otherwise returns True if file has been written,False if it hasn't.
    File won't be written if it has no content.
    """
    # Parse XSLT
    try:
        style_filename = get_stylesheet(stylesheet)
        style = etree.parse(style_filename)

        document = copy.deepcopy(wldoc)
        del wldoc
        document.swap_endlines()

        if flags:
            for flag in flags:
                document.edoc.getroot().set(flag, 'yes')

        document.clean_ed_note()

        if not options:
            options = {}
        result = document.transform(style, **options)
        del document # no longer needed large object :)

        if html_has_content(result):
            add_anchors(result.getroot())
            add_table_of_themes(result.getroot())
            add_table_of_contents(result.getroot())

            return OutputFile.from_string(etree.tostring(result, method='html',
                xml_declaration=False, pretty_print=True, encoding='utf-8'))
        else:
            return None
    except KeyError:
        raise ValueError("'%s' is not a valid stylesheet.")
    except (XMLSyntaxError, XSLTApplyError), e:
        raise ParseError(e)

class Fragment(object):
    def __init__(self, id, themes):
        super(Fragment, self).__init__()
        self.id = id
        self.themes = themes
        self.events = []

    def append(self, event, element):
        self.events.append((event, element))

    def closed_events(self):
        stack = []
        for event, element in self.events:
            if event == 'start':
                stack.append(('end', element))
            elif event == 'end':
                try:
                    stack.pop()
                except IndexError:
                    print 'CLOSED NON-OPEN TAG:', element

        stack.reverse()
        return self.events + stack

    def to_string(self):
        result = []
        for event, element in self.closed_events():
            if event == 'start':
                result.append(u'<%s %s>' % (element.tag, ' '.join('%s="%s"' % (k, v) for k, v in element.attrib.items())))
                if element.text:
                    result.append(element.text)
            elif event == 'end':
                result.append(u'</%s>' % element.tag)
                if element.tail:
                    result.append(element.tail)
            else:
                result.append(element)

        return ''.join(result)

    def __unicode__(self):
        return self.to_string()

def extract_fragments(input_filename):
    """Extracts theme fragments from input_filename."""
    open_fragments = {}
    closed_fragments = {}

    # iterparse would die on a HTML document
    parser = etree.HTMLParser(encoding='utf-8')
    buf = cStringIO.StringIO()
    buf.write(etree.tostring(etree.parse(input_filename, parser).getroot()[0][0], encoding='utf-8'))
    buf.seek(0)

    for event, element in etree.iterparse(buf, events=('start', 'end')):
        # Process begin and end elements
        if element.get('class', '') in ('theme-begin', 'theme-end'):
            if not event == 'end': continue # Process elements only once, on end event

            # Open new fragment
            if element.get('class', '') == 'theme-begin':
                fragment = Fragment(id=element.get('fid'), themes=element.text)

                # Append parents
                if element.getparent().get('id', None) != 'book-text':
                    parents = [element.getparent()]
                    while parents[-1].getparent().get('id', None) != 'book-text':
                        parents.append(parents[-1].getparent())

                    parents.reverse()
                    for parent in parents:
                        fragment.append('start', parent)

                open_fragments[fragment.id] = fragment

            # Close existing fragment
            else:
                try:
                    fragment = open_fragments[element.get('fid')]
                except KeyError:
                    print '%s:closed not open fragment #%s' % (input_filename, element.get('fid'))
                else:
                    closed_fragments[fragment.id] = fragment
                    del open_fragments[fragment.id]

            # Append element tail to lost_text (we don't want to lose any text)
            if element.tail:
                for fragment_id in open_fragments:
                    open_fragments[fragment_id].append('text', element.tail)

        # Process all elements except begin and end
        else:
            # Omit annotation tags
            if (len(element.get('name', '')) or
                    element.get('class', '') in ('annotation', 'anchor')):
                if event == 'end' and element.tail:
                    for fragment_id in open_fragments:
                        open_fragments[fragment_id].append('text', element.tail)
            else:
                for fragment_id in open_fragments:
                    open_fragments[fragment_id].append(event, copy.copy(element))

    return closed_fragments, open_fragments

def add_anchor(element, prefix, with_link=True, with_target=True, link_text=None):
    if with_link:
        if link_text is None:
            link_text = prefix
        anchor = etree.Element('a', href='#%s' % prefix)
        anchor.set('class', 'anchor')
        anchor.text = unicode(link_text)
        if element.text:
            anchor.tail = element.text
            element.text = u''
        element.insert(0, anchor)

    if with_target:
        anchor_target = etree.Element('a', name='%s' % prefix)
        anchor_target.set('class', 'target')
        anchor_target.text = u' '
        if element.text:
            anchor_target.tail = element.text
            element.text = u''
        element.insert(0, anchor_target)

def any_ancestor(element, test):
    for ancestor in element.iterancestors():
        if test(ancestor):
            return True
    return False

def add_anchors(root):
    counter = 1
    for element in root.iterdescendants():
        if any_ancestor(element, lambda e: e.get('class') in ('note', 'motto', 'motto_podpis', 'dedication')
        or e.get('id') == 'nota_red'
        or e.tag == 'blockquote'):
            continue

        if element.tag == 'p' and 'verse' in element.get('class', ''):
            if counter == 1 or counter % 5 == 0:
                add_anchor(element, "f%d" % counter, link_text=counter)
            counter += 1
        elif 'paragraph' in element.get('class', ''):
            add_anchor(element, "f%d" % counter, link_text=counter)
            counter += 1

def raw_printable_text(element):
    working = copy.deepcopy(element)
    for e in working.findall('a'):
        if e.get('class') == 'annotation':
            e.text = ''
    return etree.tostring(working, method='text', encoding=unicode).strip()

def add_table_of_contents(root):
    sections = []
    counter = 1
    for element in root.iterdescendants():
        if element.tag in ('h2', 'h3'):
            if any_ancestor(element, lambda e: e.get('id') in ('footnotes', 'nota_red') or e.get('class') in ('person-list',)):
                continue

            element_text = raw_printable_text(element)
            if element.tag == 'h3' and len(sections) and sections[-1][1] == 'h2':
                sections[-1][3].append((counter, element.tag, element_text, []))
            else:
                sections.append((counter, element.tag, element_text, []))
            add_anchor(element, "s%d" % counter, with_link=False)
            counter += 1

    toc = etree.Element('div')
    toc.set('id', 'toc')
    toc_header = etree.SubElement(toc, 'h2')
    toc_header.text = u'Spis treści'
    toc_list = etree.SubElement(toc, 'ol')

    for n, section, text, subsections in sections:
        section_element = etree.SubElement(toc_list, 'li')
        add_anchor(section_element, "s%d" % n, with_target=False, link_text=text)

        if len(subsections):
            subsection_list = etree.SubElement(section_element, 'ol')
            for n, subsection, text, _ in subsections:
                subsection_element = etree.SubElement(subsection_list, 'li')
                add_anchor(subsection_element, "s%d" % n, with_target=False, link_text=text)

    root.insert(0, toc)

def add_table_of_themes(root):
    try:
        from sortify import sortify
    except ImportError:
        sortify = lambda x: x

    book_themes = {}
    for fragment in root.findall('.//a[@class="theme-begin"]'):
        if not fragment.text:
            continue
        theme_names = [s.strip() for s in fragment.text.split(',')]
        for theme_name in theme_names:
            book_themes.setdefault(theme_name, []).append(fragment.get('name'))
    book_themes = book_themes.items()
    book_themes.sort(key=lambda s: sortify(s[0]))
    themes_div = etree.Element('div', id="themes")
    themes_ol = etree.SubElement(themes_div, 'ol')
    for theme_name, fragments in book_themes:
        themes_li = etree.SubElement(themes_ol, 'li')
        themes_li.text = "%s: " % theme_name
        for i, fragment in enumerate(fragments):
            item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
            item.text = str(i + 1)
    root.insert(0, themes_div)

def extract_annotations(html_path):
    """For each annotation, yields a tuple: anchor, text, html."""
    parser = etree.HTMLParser(encoding='utf-8')
    tree = etree.parse(html_path, parser)
    footnotes = tree.find('//*[@id="footnotes"]')
    if footnotes is not None:
        for footnote in footnotes.findall('div'):
            anchor = footnote.find('a[@name]').get('name')
            del footnote[:2]
            text_str = etree.tostring(footnote, method='text', encoding='utf-8').strip()
            html_str = etree.tostring(footnote, method='html', encoding='utf-8')
            yield anchor, text_str, html_str

<commit_msg>Spaces in themes list in HTML.<commit_after># -*- coding: utf-8 -*-
#
# This file is part of Librarian, licensed under GNU Affero GPLv3 or later.
# Copyright © Fundacja Nowoczesna Polska. See NOTICE for more information.
#
import os
import cStringIO
import copy

from lxml import etree
from librarian import XHTMLNS, ParseError, OutputFile
from librarian import functions

from lxml.etree import XMLSyntaxError, XSLTApplyError

functions.reg_substitute_entities()
functions.reg_person_name()

STYLESHEETS = {
    'legacy': 'xslt/book2html.xslt',
    'full': 'xslt/wl2html_full.xslt',
    'partial': 'xslt/wl2html_partial.xslt'
}

def get_stylesheet(name):
    return os.path.join(os.path.dirname(__file__), STYLESHEETS[name])

def html_has_content(text):
    return etree.ETXPath('//p|//{%(ns)s}p|//h1|//{%(ns)s}h1' % {'ns': str(XHTMLNS)})(text)

def transform(wldoc, stylesheet='legacy', options=None, flags=None):
    """Transforms the WL document to XHTML.

    If output_filename is None, returns an XML,
    otherwise returns True if file has been written,False if it hasn't.
    File won't be written if it has no content.
    """
    # Parse XSLT
    try:
        style_filename = get_stylesheet(stylesheet)
        style = etree.parse(style_filename)

        document = copy.deepcopy(wldoc)
        del wldoc
        document.swap_endlines()

        if flags:
            for flag in flags:
                document.edoc.getroot().set(flag, 'yes')

        document.clean_ed_note()

        if not options:
            options = {}
        result = document.transform(style, **options)
        del document # no longer needed large object :)

        if html_has_content(result):
            add_anchors(result.getroot())
            add_table_of_themes(result.getroot())
            add_table_of_contents(result.getroot())

            return OutputFile.from_string(etree.tostring(result, method='html',
                xml_declaration=False, pretty_print=True, encoding='utf-8'))
        else:
            return None
    except KeyError:
        raise ValueError("'%s' is not a valid stylesheet.")
    except (XMLSyntaxError, XSLTApplyError), e:
        raise ParseError(e)

class Fragment(object):
    def __init__(self, id, themes):
        super(Fragment, self).__init__()
        self.id = id
        self.themes = themes
        self.events = []

    def append(self, event, element):
        self.events.append((event, element))

    def closed_events(self):
        stack = []
        for event, element in self.events:
            if event == 'start':
                stack.append(('end', element))
            elif event == 'end':
                try:
                    stack.pop()
                except IndexError:
                    print 'CLOSED NON-OPEN TAG:', element

        stack.reverse()
        return self.events + stack

    def to_string(self):
        result = []
        for event, element in self.closed_events():
            if event == 'start':
                result.append(u'<%s %s>' % (element.tag, ' '.join('%s="%s"' % (k, v) for k, v in element.attrib.items())))
                if element.text:
                    result.append(element.text)
            elif event == 'end':
                result.append(u'</%s>' % element.tag)
                if element.tail:
                    result.append(element.tail)
            else:
                result.append(element)

        return ''.join(result)

    def __unicode__(self):
        return self.to_string()

def extract_fragments(input_filename):
    """Extracts theme fragments from input_filename."""
    open_fragments = {}
    closed_fragments = {}

    # iterparse would die on a HTML document
    parser = etree.HTMLParser(encoding='utf-8')
    buf = cStringIO.StringIO()
    buf.write(etree.tostring(etree.parse(input_filename, parser).getroot()[0][0], encoding='utf-8'))
    buf.seek(0)

    for event, element in etree.iterparse(buf, events=('start', 'end')):
        # Process begin and end elements
        if element.get('class', '') in ('theme-begin', 'theme-end'):
            if not event == 'end': continue # Process elements only once, on end event

            # Open new fragment
            if element.get('class', '') == 'theme-begin':
                fragment = Fragment(id=element.get('fid'), themes=element.text)

                # Append parents
                if element.getparent().get('id', None) != 'book-text':
                    parents = [element.getparent()]
                    while parents[-1].getparent().get('id', None) != 'book-text':
                        parents.append(parents[-1].getparent())

                    parents.reverse()
                    for parent in parents:
                        fragment.append('start', parent)

                open_fragments[fragment.id] = fragment

            # Close existing fragment
            else:
                try:
                    fragment = open_fragments[element.get('fid')]
                except KeyError:
                    print '%s:closed not open fragment #%s' % (input_filename, element.get('fid'))
                else:
                    closed_fragments[fragment.id] = fragment
                    del open_fragments[fragment.id]

            # Append element tail to lost_text (we don't want to lose any text)
            if element.tail:
                for fragment_id in open_fragments:
                    open_fragments[fragment_id].append('text', element.tail)

        # Process all elements except begin and end
        else:
            # Omit annotation tags
            if (len(element.get('name', '')) or
                    element.get('class', '') in ('annotation', 'anchor')):
                if event == 'end' and element.tail:
                    for fragment_id in open_fragments:
                        open_fragments[fragment_id].append('text', element.tail)
            else:
                for fragment_id in open_fragments:
                    open_fragments[fragment_id].append(event, copy.copy(element))

    return closed_fragments, open_fragments

def add_anchor(element, prefix, with_link=True, with_target=True, link_text=None):
    if with_link:
        if link_text is None:
            link_text = prefix
        anchor = etree.Element('a', href='#%s' % prefix)
        anchor.set('class', 'anchor')
        anchor.text = unicode(link_text)
        if element.text:
            anchor.tail = element.text
            element.text = u''
        element.insert(0, anchor)

    if with_target:
        anchor_target = etree.Element('a', name='%s' % prefix)
        anchor_target.set('class', 'target')
        anchor_target.text = u' '
        if element.text:
            anchor_target.tail = element.text
            element.text = u''
        element.insert(0, anchor_target)

def any_ancestor(element, test):
    for ancestor in element.iterancestors():
        if test(ancestor):
            return True
    return False

def add_anchors(root):
    counter = 1
    for element in root.iterdescendants():
        if any_ancestor(element, lambda e: e.get('class') in ('note', 'motto', 'motto_podpis', 'dedication')
        or e.get('id') == 'nota_red'
        or e.tag == 'blockquote'):
            continue

        if element.tag == 'p' and 'verse' in element.get('class', ''):
            if counter == 1 or counter % 5 == 0:
                add_anchor(element, "f%d" % counter, link_text=counter)
            counter += 1
        elif 'paragraph' in element.get('class', ''):
            add_anchor(element, "f%d" % counter, link_text=counter)
            counter += 1

def raw_printable_text(element):
    working = copy.deepcopy(element)
    for e in working.findall('a'):
        if e.get('class') == 'annotation':
            e.text = ''
    return etree.tostring(working, method='text', encoding=unicode).strip()

def add_table_of_contents(root):
    sections = []
    counter = 1
    for element in root.iterdescendants():
        if element.tag in ('h2', 'h3'):
            if any_ancestor(element, lambda e: e.get('id') in ('footnotes', 'nota_red') or e.get('class') in ('person-list',)):
                continue

            element_text = raw_printable_text(element)
            if element.tag == 'h3' and len(sections) and sections[-1][1] == 'h2':
                sections[-1][3].append((counter, element.tag, element_text, []))
            else:
                sections.append((counter, element.tag, element_text, []))
            add_anchor(element, "s%d" % counter, with_link=False)
            counter += 1

    toc = etree.Element('div')
    toc.set('id', 'toc')
    toc_header = etree.SubElement(toc, 'h2')
    toc_header.text = u'Spis treści'
    toc_list = etree.SubElement(toc, 'ol')

    for n, section, text, subsections in sections:
        section_element = etree.SubElement(toc_list, 'li')
        add_anchor(section_element, "s%d" % n, with_target=False, link_text=text)

        if len(subsections):
            subsection_list = etree.SubElement(section_element, 'ol')
            for n, subsection, text, _ in subsections:
                subsection_element = etree.SubElement(subsection_list, 'li')
                add_anchor(subsection_element, "s%d" % n, with_target=False, link_text=text)

    root.insert(0, toc)

def add_table_of_themes(root):
    try:
        from sortify import sortify
    except ImportError:
        sortify = lambda x: x

    book_themes = {}
    for fragment in root.findall('.//a[@class="theme-begin"]'):
        if not fragment.text:
            continue
        theme_names = [s.strip() for s in fragment.text.split(',')]
        for theme_name in theme_names:
            book_themes.setdefault(theme_name, []).append(fragment.get('name'))
    book_themes = book_themes.items()
    book_themes.sort(key=lambda s: sortify(s[0]))
    themes_div = etree.Element('div', id="themes")
    themes_ol = etree.SubElement(themes_div, 'ol')
    for theme_name, fragments in book_themes:
        themes_li = etree.SubElement(themes_ol, 'li')
        themes_li.text = "%s: " % theme_name
        for i, fragment in enumerate(fragments):
            item = etree.SubElement(themes_li, 'a', href="#%s" % fragment)
            item.text = str(i + 1)
            item.tail = ' '
    root.insert(0, themes_div)

def extract_annotations(html_path):
    """For each annotation, yields a tuple: anchor, text, html."""
    parser = etree.HTMLParser(encoding='utf-8')
    tree = etree.parse(html_path, parser)
    footnotes = tree.find('//*[@id="footnotes"]')
    if footnotes is not None:
        for footnote in footnotes.findall('div'):
            anchor = footnote.find('a[@name]').get('name')
            del footnote[:2]
            text_str = etree.tostring(footnote, method='text', encoding='utf-8').strip()
            html_str = etree.tostring(footnote, method='html', encoding='utf-8')
            yield anchor, text_str, html_str

<|endoftext|>


<commit_before>"""Contains sound-related asset classes used by the audio system"""

import logging

from mpf.core.assets import Asset, AssetPool
from mpf.core.utility_functions import Util
from mpfmc.core.audio.audio_interface import AudioInterface, AudioException

Logger = logging.getLogger('SoundAsset')

# ---------------------------------------------------------------------------
#    Default sound asset configuration parameter values
# ---------------------------------------------------------------------------
DEFAULT_VOLUME = 0.5
DEFAULT_PRIORITY = 0
DEFAULT_MAX_QUEUE_TIME = None
DEFAULT_LOOPS = 0
MINIMUM_DUCKING_DURATION = "10ms"

class SoundPool(AssetPool):
    """Allows several Sound assets to be grouped together in a 'pool' and referenced
    as a single sound when playing back.  Allows for easily managed variations
    of a sound.
    """
    # Be sure the pool group, if you use it, is first in the file ahead of the
    # asset class.

    def __repr__(self):
        """String that's returned if someone prints this object"""
        return '<SoundPool: {}>'.format(self.name)

    @property
    def sound(self):
        """The currently selected Sound object from the pool"""
        return self.asset

    def play(self, settings=None):
        """
        Plays the sound using the specified settings
        Args:
            settings: Optional dictionary of settings to override the default values.
        """
        self.sound.play(settings)

    def stop(self):
        """
        Stops all instances of all sounds contained in the sound pool.
        """
        for sound in self.assets:
            sound.stop()

class SoundAsset(Asset):
    """
    Sound asset class contains a single sound that may be played using the audio engine.

    Notes:
        It is critical that the AudioInterface be initialized before any Sound assets
        are loaded.  The loading code relies upon having an active audio interface.
    """
    attribute = 'sounds'  # attribute in MC, e.g. self.mc.images
    path_string = 'sounds'  # entry from mpf_mc:paths: for asset folder name
    config_section = 'sounds'  # section in the config files for this asset
    extensions = ('wav',)  # Additional extensions may be added at runtime ('ogg',
    # 'flac') depending upon the SDL_Mixer plug-ins installed on the system
    class_priority = 100  # Order asset classes will be loaded. Higher is first.
    pool_config_section = 'sound_pools'  # Will setup groups if present
    asset_group_class = SoundPool  # Class or None to not use pools

    def __init__(self, mc, name, file, config):
        """ Constructor"""
        super().__init__(mc, name, file, config)

        self._track = None
        self._volume = DEFAULT_VOLUME
        self.priority = DEFAULT_PRIORITY
        self._max_queue_time = DEFAULT_MAX_QUEUE_TIME
        self._loops = DEFAULT_LOOPS
        self._events_when_played = None
        self._events_when_stopped = None
        self._container = None  # holds the actual sound samples in memory
        self._ducking = None

        # Make sure a legal track name has been specified.  If not, throw an exception
        track = self.machine.sound_system.audio_interface.get_track_by_name(self.config['track'])
        if 'track' not in self.config or track is None:
            Logger.error("SoundAsset: sound must have a valid track name. "
                         "Could not create sound '%s' asset.", name)
            raise AudioException("Sound must have a valid track name. "
                                 "Could not create sound '{}' asset".format(name))

        self._track = track

        # Validate sound attributes and provide default values
        if 'volume' in self.config:
            self._volume = min(max(float(self.config['volume']), 0.0), 1.0)

        if 'priority' in self.config:
            self.priority = int(self.config['priority'])

        if 'max_queue_time' in self.config and self.config['max_queue_time'] is not None:
            self._max_queue_time = AudioInterface.string_to_secs(self.config['max_queue_time'])

        if 'loops' in self.config:
            self._loops = int(self.config['loops'])

        if 'events_when_played' in self.config and isinstance(
                self.config['events_when_played'], str):
            self._events_when_played = Util.string_to_list(self.config['events_when_played'])

        if 'events_when_stopped' in self.config and isinstance(
                self.config['events_when_stopped'], str):
            self._events_when_stopped = Util.string_to_list(self.config['events_when_stopped'])

        if 'ducking' in self.config:
            self._ducking = DuckingSettings(self.machine, self.config['ducking'])

            # An attenuation value of exactly 1.0 does absolutely nothing so
            # there is no point in keeping the ducking settings for this
            # sound when attenuation is 1.0.
            if self._ducking.attenuation == 1.0:
                self._ducking = None

        # Add sound to a dictionary of sound objects keyed by sound id
        if not hasattr(self.machine, 'sounds_by_id'):
            setattr(self.machine, 'sounds_by_id', dict())

        self.machine.sounds_by_id[self.id] = self

    def __repr__(self):
        """String that's returned if someone prints this object"""
        return '<Sound: {}({}), Loaded={}>'.format(self.name, self.id, self.loaded)

    def __lt__(self, other):
        """Less than comparison operator"""
        # Note this is "backwards" (It's the __lt__ method but the formula uses
        # greater than because the PriorityQueue puts lowest first.)
        if other is None:
            return False
        else:
            return ("%s, %s" % (self.priority, self._id) >
                    "%s, %s" % (other.priority, other.get_id()))

    @property
    def id(self):
        """
        The id property contains a unique identifier for the sound (based on the Python id()
        function).  This id is used in the audio interface to uniquely identify a sound
        (rather than the name) due to the hassle of passing strings between Python and Cython.
        Returns:
            An integer uniquely identifying the sound
        """
        return id(self)

    @property
    def track(self):
        """The track object on which to play the sound"""
        return self._track

    @property
    def volume(self):
        """Returns the volume of the sound (float 0.0 to 1.0)"""
        return self._volume

    @property
    def max_queue_time(self):
        """Returns the maximum time a sound may be queued before
        playing or being discarded"""
        return self._max_queue_time

    @property
    def loops(self):
        """Returns the looping setting for the sound.
        0 - do not loop, -1 loop infinitely, >= 1 the number of
        times to loop."""
        return self._loops

    @property
    def events_when_played(self):
        """Returns the list of events that are posted when the sound is played"""
        return self._events_when_played

    @property
    def events_when_stopped(self):
        """Returns the list of events that are posted when the sound is stopped"""
        return self._events_when_stopped

    @property
    def container(self):
        """The container object wrapping the SDL structure containing the actual sound data"""
        return self._container

    @property
    def ducking(self):
        """A DuckingSettings object containing the ducking settings for this sound (optional)"""
        return self._ducking

    @property
    def has_ducking(self):
        """Returns whether or not this sound has ducking"""
        return self._ducking is not None

    def do_load(self):
        """Loads the sound asset from disk."""

        # Load the sound file into memory
        self._container = AudioInterface.load_sound(self.file)

        # Validate ducking now that the sound has been loaded
        if self._ducking is not None:
            try:
                self._ducking.validate(self._container.length)
            except AudioException as exception:
                Logger.error("SoundAsset: Ducking settings for sound %s are not valid: %s",
                             self.name, str(exception))
                raise

    def _do_unload(self):
        """Unloads the asset from memory"""

        AudioInterface.unload_sound(self._container)
        self._container = None

    def is_loaded(self):
        """Called when the asset has finished loading"""
        super().is_loaded()
        Logger.debug("SoundAsset: Loaded %s (Track %s)", self.name, self.track)

    def play(self, settings=None):
        """
        Plays the sound using the specified settings
        Args:
            settings: Optional dictionary of settings to override the default values.
        """
        Logger.debug("SoundAsset: Play sound %s %s", self.name, self.track)
        self._track.play_sound(self, **settings)

    def stop(self):
        """Stops all instances of the sound playing on the sound's default track."""
        Logger.debug("SoundAsset: Stop sound %s %s", self.name, self.track)
        self._track.stop_sound(self)

    def stop_looping(self):
        """Stops looping on all instances of the sound playing (and awaiting playback)."""
        Logger.debug("SoundAsset: Stop looping sound %s %s", self.name, self.track)
        self._track.stop_sound_looping(self)

class DuckingSettings(object):
    """ DuckingSettings contains the parameters needed to control audio ducking
    for a sound.
    """

    def __init__(self, mc, config):
        """
        Constructor
        Args:
            mc: The media controller instance.
            config: The ducking configuration file section that contains all the ducking
                settings for the sound.

        Notes:
            The config section should contain the following attributes:
                target: The track name to apply the ducking to when the sound is played.
                delay: The duration (in samples) of the delay period (time before attack starts)
                attack: The duration (in samples) of the attack stage of the ducking envelope
                attenuation: The attenuation (gain) (0.0 to 1.0) to apply to the target track while
                    ducking
                release_point: The point (in samples) relative to the end of the sound at which
                    to start the release stage.  A positive value indicates prior to the end of
                    the sound while a negative value indicates to start the release after the
                    end of the sound.
                release: The duration (in samples) of the release stage of the ducking process.
        """
        if config is None:
            raise AudioException("The 'ducking' configuration must include the following "
                                 "attributes: track, delay, attack, attenuation, "
                                 "release_point, and release")

        if 'target' not in config:
            raise AudioException("'ducking.target' must contain at least one "
                                 "valid audio track name")

        self._track = mc.sound_system.audio_interface.get_track_by_name(config['target'])
        if self._track is None:
            raise AudioException("'ducking.target' must contain a valid audio track name")

        # Delay is optional (defaults to 0, must be >= 0)
        if 'delay' in config:
            self._delay = max(mc.sound_system.audio_interface.string_to_samples(
                config['delay']), 0)
        else:
            self._delay = 0

        if 'attack' not in config:
            raise AudioException("'ducking.attack' must contain a valid attack value (time "
                                 "string or number of samples)")
        self._attack = max(mc.sound_system.audio_interface.string_to_samples(config['attack']),
                           mc.sound_system.audio_interface.string_to_samples(
                               MINIMUM_DUCKING_DURATION))

        if 'attenuation' not in config:
            raise AudioException("'ducking.attenuation' must contain valid attenuation "
                                 "value (0.0 to 1.0)")
        self._attenuation = min(max(float(AudioInterface.string_to_gain(
            config['attenuation'])), 0.0), 1.0)

        if 'release_point' not in config:
            raise AudioException("'ducking.release_point' must contain a valid release point "
                                 "value (time string or number of samples)")
        # Release point cannot be negative (must be before or at the end of the sound)
        self._release_point = max(mc.sound_system.audio_interface.string_to_samples(
            config['release_point']), 0)

        if 'release' not in config:
            raise AudioException("'ducking.release' must contain a valid release "
                                 "value (time string or number of samples)")
        self._release = max(mc.sound_system.audio_interface.string_to_samples(config['release']),
                            mc.sound_system.audio_interface.string_to_samples(
                                MINIMUM_DUCKING_DURATION))

    @property
    def track(self):
        """ The track object to apply the ducking envelope"""
        return self._track

    @property
    def delay(self):
        """The duration (in samples) of the delay period (time before attack starts)"""
        return self._delay

    @property
    def attack(self):
        """The duration (in samples) of the attack stage of the ducking envelope"""
        return self._attack

    @property
    def attenuation(self):
        """The attenuation (gain) (0.0 to 1.0) to apply to the target track while ducking"""
        return self._attenuation

    @property
    def release_point(self):
        """The point (in samples) relative to the end of the sound at which
        to start the release stage.  A positive value indicates prior to the end of
        the sound while a negative value indicates to start the release after the
        end of the sound.
        """
        return self._release_point

    @property
    def release(self):
        """The duration (in samples) of the release stage of the ducking process."""
        return self._release

    def validate(self, sound_length):
        """
        Validates the ducking settings against the length of the sound to ensure all
        settings are valid.
        Args:
            sound_length: The length of the sound in samples

        Returns:
            True if all settings are valid, otherwise an exception will be thrown
        """
        if sound_length is None or sound_length == 0:
            raise AudioException("ducking may not be applied to an empty/zero length sound")

        if self._attack > sound_length:
            raise AudioException("'ducking.attack' value may not be longer than the "
                                 "length of the sound")

        if self._release_point >= sound_length:
            raise AudioException("'ducking.release_point' value may not occur before the "
                                 "beginning of the sound")

        if self._release_point + self._attack >= sound_length:
            raise AudioException("'ducking.release_point' value may not occur before "
                                 "the ducking attack segment has completed")

        return True
<commit_msg>Further logging improvements<commit_after>"""Contains sound-related asset classes used by the audio system"""

import logging

from mpf.core.assets import Asset, AssetPool
from mpf.core.utility_functions import Util
from mpfmc.core.audio.audio_interface import AudioInterface, AudioException

# ---------------------------------------------------------------------------
#    Default sound asset configuration parameter values
# ---------------------------------------------------------------------------
DEFAULT_VOLUME = 0.5
DEFAULT_PRIORITY = 0
DEFAULT_MAX_QUEUE_TIME = None
DEFAULT_LOOPS = 0
MINIMUM_DUCKING_DURATION = "10ms"

class SoundPool(AssetPool):
    """Allows several Sound assets to be grouped together in a 'pool' and referenced
    as a single sound when playing back.  Allows for easily managed variations
    of a sound.
    """
    # Be sure the pool group, if you use it, is first in the file ahead of the
    # asset class.

    def __repr__(self):
        """String that's returned if someone prints this object"""
        return '<SoundPool: {}>'.format(self.name)

    @property
    def sound(self):
        """The currently selected Sound object from the pool"""
        return self.asset

    def play(self, settings=None):
        """
        Plays the sound using the specified settings
        Args:
            settings: Optional dictionary of settings to override the default values.
        """
        self.sound.play(settings)

    def stop(self):
        """
        Stops all instances of all sounds contained in the sound pool.
        """
        for sound in self.assets:
            sound.stop()

class SoundAsset(Asset):
    """
    Sound asset class contains a single sound that may be played using the audio engine.

    Notes:
        It is critical that the AudioInterface be initialized before any Sound assets
        are loaded.  The loading code relies upon having an active audio interface.
    """
    attribute = 'sounds'  # attribute in MC, e.g. self.mc.images
    path_string = 'sounds'  # entry from mpf_mc:paths: for asset folder name
    config_section = 'sounds'  # section in the config files for this asset
    extensions = ('wav',)  # Additional extensions may be added at runtime ('ogg',
    # 'flac') depending upon the SDL_Mixer plug-ins installed on the system
    class_priority = 100  # Order asset classes will be loaded. Higher is first.
    pool_config_section = 'sound_pools'  # Will setup groups if present
    asset_group_class = SoundPool  # Class or None to not use pools

    def __init__(self, mc, name, file, config):
        """ Constructor"""
        super().__init__(mc, name, file, config)

        self._track = None
        self._volume = DEFAULT_VOLUME
        self.priority = DEFAULT_PRIORITY
        self._max_queue_time = DEFAULT_MAX_QUEUE_TIME
        self._loops = DEFAULT_LOOPS
        self._events_when_played = None
        self._events_when_stopped = None
        self._container = None  # holds the actual sound samples in memory
        self._ducking = None
        self.log = logging.getLogger('SoundAsset')

        # Make sure a legal track name has been specified.  If not, throw an exception
        track = self.machine.sound_system.audio_interface.get_track_by_name(self.config['track'])
        if 'track' not in self.config or track is None:
            self.log.error("sound must have a valid track name. "
                           "Could not create sound '%s' asset.", name)
            raise AudioException("Sound must have a valid track name. "
                                 "Could not create sound '{}' asset".format(name))

        self._track = track

        # Validate sound attributes and provide default values
        if 'volume' in self.config:
            self._volume = min(max(float(self.config['volume']), 0.0), 1.0)

        if 'priority' in self.config:
            self.priority = int(self.config['priority'])

        if 'max_queue_time' in self.config and self.config['max_queue_time'] is not None:
            self._max_queue_time = AudioInterface.string_to_secs(self.config['max_queue_time'])

        if 'loops' in self.config:
            self._loops = int(self.config['loops'])

        if 'events_when_played' in self.config and isinstance(
                self.config['events_when_played'], str):
            self._events_when_played = Util.string_to_list(self.config['events_when_played'])

        if 'events_when_stopped' in self.config and isinstance(
                self.config['events_when_stopped'], str):
            self._events_when_stopped = Util.string_to_list(self.config['events_when_stopped'])

        if 'ducking' in self.config:
            self._ducking = DuckingSettings(self.machine, self.config['ducking'])

            # An attenuation value of exactly 1.0 does absolutely nothing so
            # there is no point in keeping the ducking settings for this
            # sound when attenuation is 1.0.
            if self._ducking.attenuation == 1.0:
                self._ducking = None

        # Add sound to a dictionary of sound objects keyed by sound id
        if not hasattr(self.machine, 'sounds_by_id'):
            setattr(self.machine, 'sounds_by_id', dict())

        self.machine.sounds_by_id[self.id] = self

    def __repr__(self):
        """String that's returned if someone prints this object"""
        return '<Sound: {}({}), Loaded={}>'.format(self.name, self.id, self.loaded)

    def __lt__(self, other):
        """Less than comparison operator"""
        # Note this is "backwards" (It's the __lt__ method but the formula uses
        # greater than because the PriorityQueue puts lowest first.)
        if other is None:
            return False
        else:
            return ("%s, %s" % (self.priority, self._id) >
                    "%s, %s" % (other.priority, other.get_id()))

    @property
    def id(self):
        """
        The id property contains a unique identifier for the sound (based on the Python id()
        function).  This id is used in the audio interface to uniquely identify a sound
        (rather than the name) due to the hassle of passing strings between Python and Cython.
        Returns:
            An integer uniquely identifying the sound
        """
        return id(self)

    @property
    def track(self):
        """The track object on which to play the sound"""
        return self._track

    @property
    def volume(self):
        """Returns the volume of the sound (float 0.0 to 1.0)"""
        return self._volume

    @property
    def max_queue_time(self):
        """Returns the maximum time a sound may be queued before
        playing or being discarded"""
        return self._max_queue_time

    @property
    def loops(self):
        """Returns the looping setting for the sound.
        0 - do not loop, -1 loop infinitely, >= 1 the number of
        times to loop."""
        return self._loops

    @property
    def events_when_played(self):
        """Returns the list of events that are posted when the sound is played"""
        return self._events_when_played

    @property
    def events_when_stopped(self):
        """Returns the list of events that are posted when the sound is stopped"""
        return self._events_when_stopped

    @property
    def container(self):
        """The container object wrapping the SDL structure containing the actual sound data"""
        return self._container

    @property
    def ducking(self):
        """A DuckingSettings object containing the ducking settings for this sound (optional)"""
        return self._ducking

    @property
    def has_ducking(self):
        """Returns whether or not this sound has ducking"""
        return self._ducking is not None

    def do_load(self):
        """Loads the sound asset from disk."""

        # Load the sound file into memory
        self._container = AudioInterface.load_sound(self.file)

        # Validate ducking now that the sound has been loaded
        if self._ducking is not None:
            try:
                self._ducking.validate(self._container.length)
            except AudioException as exception:
                self.log.error("Ducking settings for sound %s are not valid: %s",
                               self.name, str(exception))
                raise

    def _do_unload(self):
        """Unloads the asset from memory"""

        AudioInterface.unload_sound(self._container)
        self._container = None

    def is_loaded(self):
        """Called when the asset has finished loading"""
        super().is_loaded()
        self.log.debug("Loaded %s (Track %s)", self.name, self.track)

    def play(self, settings=None):
        """
        Plays the sound using the specified settings
        Args:
            settings: Optional dictionary of settings to override the default values.
        """
        self.log.debug("Play sound %s %s", self.name, self.track)
        self._track.play_sound(self, **settings)

    def stop(self):
        """Stops all instances of the sound playing on the sound's default track."""
        self.log.debug("Stop sound %s %s", self.name, self.track)
        self._track.stop_sound(self)

    def stop_looping(self):
        """Stops looping on all instances of the sound playing (and awaiting playback)."""
        self.log.debug("Stop looping sound %s %s", self.name, self.track)
        self._track.stop_sound_looping(self)

class DuckingSettings(object):
    """ DuckingSettings contains the parameters needed to control audio ducking
    for a sound.
    """

    def __init__(self, mc, config):
        """
        Constructor
        Args:
            mc: The media controller instance.
            config: The ducking configuration file section that contains all the ducking
                settings for the sound.

        Notes:
            The config section should contain the following attributes:
                target: The track name to apply the ducking to when the sound is played.
                delay: The duration (in samples) of the delay period (time before attack starts)
                attack: The duration (in samples) of the attack stage of the ducking envelope
                attenuation: The attenuation (gain) (0.0 to 1.0) to apply to the target track while
                    ducking
                release_point: The point (in samples) relative to the end of the sound at which
                    to start the release stage.  A positive value indicates prior to the end of
                    the sound while a negative value indicates to start the release after the
                    end of the sound.
                release: The duration (in samples) of the release stage of the ducking process.
        """
        if config is None:
            raise AudioException("The 'ducking' configuration must include the following "
                                 "attributes: track, delay, attack, attenuation, "
                                 "release_point, and release")

        if 'target' not in config:
            raise AudioException("'ducking.target' must contain at least one "
                                 "valid audio track name")

        self._track = mc.sound_system.audio_interface.get_track_by_name(config['target'])
        if self._track is None:
            raise AudioException("'ducking.target' must contain a valid audio track name")

        # Delay is optional (defaults to 0, must be >= 0)
        if 'delay' in config:
            self._delay = max(mc.sound_system.audio_interface.string_to_samples(
                config['delay']), 0)
        else:
            self._delay = 0

        if 'attack' not in config:
            raise AudioException("'ducking.attack' must contain a valid attack value (time "
                                 "string or number of samples)")
        self._attack = max(mc.sound_system.audio_interface.string_to_samples(config['attack']),
                           mc.sound_system.audio_interface.string_to_samples(
                               MINIMUM_DUCKING_DURATION))

        if 'attenuation' not in config:
            raise AudioException("'ducking.attenuation' must contain valid attenuation "
                                 "value (0.0 to 1.0)")
        self._attenuation = min(max(float(AudioInterface.string_to_gain(
            config['attenuation'])), 0.0), 1.0)

        if 'release_point' not in config:
            raise AudioException("'ducking.release_point' must contain a valid release point "
                                 "value (time string or number of samples)")
        # Release point cannot be negative (must be before or at the end of the sound)
        self._release_point = max(mc.sound_system.audio_interface.string_to_samples(
            config['release_point']), 0)

        if 'release' not in config:
            raise AudioException("'ducking.release' must contain a valid release "
                                 "value (time string or number of samples)")
        self._release = max(mc.sound_system.audio_interface.string_to_samples(config['release']),
                            mc.sound_system.audio_interface.string_to_samples(
                                MINIMUM_DUCKING_DURATION))

    @property
    def track(self):
        """ The track object to apply the ducking envelope"""
        return self._track

    @property
    def delay(self):
        """The duration (in samples) of the delay period (time before attack starts)"""
        return self._delay

    @property
    def attack(self):
        """The duration (in samples) of the attack stage of the ducking envelope"""
        return self._attack

    @property
    def attenuation(self):
        """The attenuation (gain) (0.0 to 1.0) to apply to the target track while ducking"""
        return self._attenuation

    @property
    def release_point(self):
        """The point (in samples) relative to the end of the sound at which
        to start the release stage.  A positive value indicates prior to the end of
        the sound while a negative value indicates to start the release after the
        end of the sound.
        """
        return self._release_point

    @property
    def release(self):
        """The duration (in samples) of the release stage of the ducking process."""
        return self._release

    def validate(self, sound_length):
        """
        Validates the ducking settings against the length of the sound to ensure all
        settings are valid.
        Args:
            sound_length: The length of the sound in samples

        Returns:
            True if all settings are valid, otherwise an exception will be thrown
        """
        if sound_length is None or sound_length == 0:
            raise AudioException("ducking may not be applied to an empty/zero length sound")

        if self._attack > sound_length:
            raise AudioException("'ducking.attack' value may not be longer than the "
                                 "length of the sound")

        if self._release_point >= sound_length:
            raise AudioException("'ducking.release_point' value may not occur before the "
                                 "beginning of the sound")

        if self._release_point + self._attack >= sound_length:
            raise AudioException("'ducking.release_point' value may not occur before "
                                 "the ducking attack segment has completed")

        return True
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division,
                        print_function)
import numpy as np
from srttools.core.interactive_filter import ImageSelector, DataSelector
from srttools.core.interactive_filter import select_data
from srttools.core.interactive_filter import TestWarning, PlotWarning
import warnings
import pytest

np.random.seed(1241347)

class TestImageSelector(object):
    @classmethod
    def setup_class(klass):
        import matplotlib.pyplot as plt
        klass.data = np.zeros((100, 100))
        klass.ax = plt.subplot()

        def fun(x, y, key):
            warnings.warn("It is working: {}, {}, {}".format(x, y, key),
                          TestWarning)
        klass.selector = ImageSelector(klass.data, klass.ax, test=True,
                                       fun=fun)

    def test_interactive_valid_data(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'q'
        fake_event.xdata, fake_event.ydata = (130, 30)

        retval = self.selector.on_key(fake_event)
        assert retval == (130, 30, 'q')

    def test_interactive_invalid_data(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'b'
        fake_event.xdata, fake_event.ydata = (None, 30)

        retval = self.selector.on_key(fake_event)
        assert retval is None

    def test_interactive_fun(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'b'
        fake_event.xdata, fake_event.ydata = (130, 30)

        with pytest.warns(TestWarning) as record:
            retval = self.selector.on_key(fake_event)
        assert "It is working: 130, 30, b" in record[0].message.args[0]
        assert retval == (130, 30, 'b')

class TestDataSelector(object):
    @classmethod
    def setup_class(klass):
        import matplotlib.pyplot as plt
        import matplotlib as mpl
        chans = ['scan1.fits', 'scan2.fits']
        klass.xs = {c: np.arange(30) for c in chans}
        klass.ys = {c: -1 ** i * np.random.normal(klass.xs[c] * 0.1, 0.1) + i
                    for i, c in enumerate(chans)}

        gs = mpl.gridspec.GridSpec(2, 1)

        klass.ax0 = plt.subplot(gs[0])
        klass.ax1 = plt.subplot(gs[1])

        klass.selector = DataSelector(klass.xs, klass.ys, klass.ax0, klass.ax1,
                                      test=True)
        klass.selector.current = 'scan1.fits'

    def test_interactive_zap(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 4, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I select a zap interval at 4" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['zap'].xs == [1, 4]
        assert self.selector.info['scan1.fits']['zap'].ys == [3, 3]
        assert self.selector.zcounter == 2

    def test_interactive_base(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 4, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I put a baseline mark at 4" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['base'].xs == [1, 4]
        assert self.selector.info['scan1.fits']['base'].ys == [3, 3]
        assert self.selector.bcounter == 2

    def test_print_instructions(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('h', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Interactive plotter." in out
        assert 'z     create zap intervals' in out

    def test_update(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('u', 1, 3)
        with pytest.warns(PlotWarning) as record:
            self.selector.on_key(fake_event)
        assert "I plotted all" in record[0].message.args[0]

    def test_flag(self, capsys):
        assert self.selector.info['scan1.fits']['FLAG'] is False
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was flagged" in out
        assert self.selector.info['scan1.fits']['FLAG'] is True

    def test_flag_otherscan(self, capsys):
        self.selector.current = 'scan2.fits'
        assert self.selector.info['scan2.fits']['FLAG'] is False
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was flagged" in out
        assert self.selector.info['scan2.fits']['FLAG'] is True
        self.selector.current = 'scan1.fits'

    def test_unflag(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('v', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was unflagged" in out
        assert self.selector.info['scan1.fits']['FLAG'] is False

    def test_reset(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        self.selector.on_key(fake_event)
        self.selector.current = 'scan2.fits'
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 1, 3)
        self.selector.on_key(fake_event)
        self.selector.current = 'scan1.fits'
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('r', 1, 3)
        self.selector.on_key(fake_event)
        assert self.selector.info['scan2.fits']['FLAG'] is False
        assert self.selector.info['scan1.fits']['base'].xs == []
        assert self.selector.info['scan1.fits']['zap'].xs == []
        assert self.selector.info['scan1.fits']['fitpars'][0] == 0

    def test_subtract_baseline_one_interval(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 4, 3)
        self.selector.on_key(fake_event)

        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('B', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I subtract" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['fitpars'][1] != 0

    def test_subtract_baseline_no_interval(self):
        # Reset all
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('r', 1, 3)
        self.selector.on_key(fake_event)
        # Then fit
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('B', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I subtract" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['fitpars'][1] == 0

    def test_align_all(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('A', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I aligned all" in record[0].message.args[0]

    def test_select_data(self):
        info = select_data(self.xs, self.ys, test=True)
        assert info['scan1.fits']['zap'].xs == []
        assert info['scan1.fits']['base'].xs == []
<commit_msg>More interactive tests<commit_after>from __future__ import (absolute_import, division,
                        print_function)
import numpy as np
from srttools.core.interactive_filter import ImageSelector, DataSelector
from srttools.core.interactive_filter import select_data
from srttools.core.interactive_filter import TestWarning, PlotWarning
import warnings
import pytest

np.random.seed(1241347)

class TestImageSelector(object):
    @classmethod
    def setup_class(klass):
        import matplotlib.pyplot as plt
        klass.data = np.zeros((100, 100))
        klass.ax = plt.subplot()

        def fun(x, y, key):
            warnings.warn("It is working: {}, {}, {}".format(x, y, key),
                          TestWarning)
        klass.selector = ImageSelector(klass.data, klass.ax, test=True,
                                       fun=fun)

    def test_interactive_valid_data(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'q'
        fake_event.xdata, fake_event.ydata = (130, 30)

        retval = self.selector.on_key(fake_event)
        assert retval == (130, 30, 'q')

    def test_interactive_invalid_data(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'b'
        fake_event.xdata, fake_event.ydata = (None, 30)

        retval = self.selector.on_key(fake_event)
        assert retval is None

    def test_interactive_fun(self):
        fake_event = type('event', (), {})()
        fake_event.key = 'b'
        fake_event.xdata, fake_event.ydata = (130, 30)

        with pytest.warns(TestWarning) as record:
            retval = self.selector.on_key(fake_event)
        assert "It is working: 130, 30, b" in record[0].message.args[0]
        assert retval == (130, 30, 'b')

class TestDataSelector(object):
    @classmethod
    def setup_class(klass):
        import matplotlib.pyplot as plt
        import matplotlib as mpl
        chans = ['scan1.fits', 'scan2.fits']
        klass.xs = {c: np.arange(30) for c in chans}
        klass.ys = {c: -1 ** i * np.random.normal(klass.xs[c] * 0.1, 0.1) + i
                    for i, c in enumerate(chans)}

        gs = mpl.gridspec.GridSpec(2, 1)

        klass.ax0 = plt.subplot(gs[0])
        klass.ax1 = plt.subplot(gs[1])

        klass.selector = DataSelector(klass.xs, klass.ys, klass.ax0, klass.ax1,
                                      test=True)
        klass.selector.current = 'scan1.fits'

    def test_interactive_zap_and_print_info(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 4, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I select a zap interval at 4" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['zap'].xs == [1, 4]
        assert self.selector.info['scan1.fits']['zap'].ys == [3, 3]
        assert self.selector.zcounter == 2
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('P', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "scan1.fits" + ":" in out
        msg_nospace = " ".join(out.split())

        assert "Zap intervals: [(1, 4)]" in msg_nospace

    def test_interactive_base(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 4, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I put a baseline mark at 4" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['base'].xs == [1, 4]
        assert self.selector.info['scan1.fits']['base'].ys == [3, 3]
        assert self.selector.bcounter == 2

    def test_print_instructions(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('h', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Interactive plotter." in out
        assert 'z     create zap intervals' in out

    def test_update(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('u', 1, 3)
        with pytest.warns(PlotWarning) as record:
            self.selector.on_key(fake_event)
        assert "I plotted all" in record[0].message.args[0]

    def test_flag(self, capsys):
        assert self.selector.info['scan1.fits']['FLAG'] is False
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was flagged" in out
        assert self.selector.info['scan1.fits']['FLAG'] is True

    def test_flag_otherscan(self, capsys):
        self.selector.current = 'scan2.fits'
        assert self.selector.info['scan2.fits']['FLAG'] is False
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was flagged" in out
        assert self.selector.info['scan2.fits']['FLAG'] is True
        self.selector.current = 'scan1.fits'

    def test_unflag(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('x', 1, 3)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('v', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Scan was unflagged" in out
        assert self.selector.info['scan1.fits']['FLAG'] is False

    def test_reset(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        self.selector.on_key(fake_event)
        self.selector.current = 'scan2.fits'
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('z', 1, 3)
        self.selector.on_key(fake_event)
        self.selector.current = 'scan1.fits'
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('r', 1, 3)
        self.selector.on_key(fake_event)
        assert self.selector.info['scan2.fits']['FLAG'] is False
        assert self.selector.info['scan1.fits']['base'].xs == []
        assert self.selector.info['scan1.fits']['zap'].xs == []
        assert self.selector.info['scan1.fits']['fitpars'][0] == 0

    def test_subtract_baseline_one_interval(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 1, 3)
        self.selector.on_key(fake_event)
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('b', 4, 3)
        self.selector.on_key(fake_event)

        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('B', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I subtract" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['fitpars'][1] != 0

    def test_subtract_baseline_no_interval(self):
        # Reset all
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('r', 1, 3)
        self.selector.on_key(fake_event)
        # Then fit
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('B', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I subtract" in record[0].message.args[0]
        assert self.selector.info['scan1.fits']['fitpars'][1] == 0

    def test_align_all(self):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('A', 1, 3)
        with pytest.warns(TestWarning) as record:
            self.selector.on_key(fake_event)
        assert "I aligned all" in record[0].message.args[0]

    def test_quit(self, capsys):
        fake_event = type('event', (), {})()
        fake_event.key, fake_event.xdata, fake_event.ydata = ('q', 1, 3)
        self.selector.on_key(fake_event)
        out, err = capsys.readouterr()
        assert "Closing all figures and quitting." in out

    def test_select_data(self):
        info = select_data(self.xs, self.ys, test=True)
        assert info['scan1.fits']['zap'].xs == []
        assert info['scan1.fits']['base'].xs == []
<|endoftext|>


<commit_before>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2015  Daniel Dietsch
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import functools
import logging
import os
import re
import subprocess

import benchexec.util as util
import benchexec.tools.template
import benchexec.result as result
from benchexec.model import MEMLIMIT

_SVCOMP17_VERSIONS = {"f7c3ed31"}
_SVCOMP17_FORBIDDEN_FLAGS = {"--full-output", "--architecture"}
_ULTIMATE_VERSION_REGEX = re.compile('^Version is (.*)$', re.MULTILINE)
_LAUNCHER_JAR = "plugins/org.eclipse.equinox.launcher_1.3.100.v20150511-1540.jar"

class UltimateTool(benchexec.tools.template.BaseTool):
    """
    Abstract tool info for Ultimate-based tools.
    """

    REQUIRED_PATHS = [
        "artifacts.xml",
        "config",
        "configuration",
        "cvc4",
        "cvc4-LICENSE",
        "features",
        "LICENSE",
        "LICENSE.GPL",
        "LICENSE.GPL.LESSER",
        "mathsat",
        "p2",
        "plugins",
        "README",
        "Ultimate",
        "Ultimate.ini",
        "Ultimate.py",
        "z3",
        "z3-LICENSE"
    ]

    REQUIRED_PATHS_SVCOMP17 = []

    def __init__(self):
        self._uses_propertyfile = False

    def executable(self):
        return util.find_executable('Ultimate.py')

    def _ultimate_version(self, executable):
        ultimatedir = os.path.dirname(executable)
        launcher_jar = os.path.join(ultimatedir, _LAUNCHER_JAR)
        data_dir = os.path.join(ultimatedir, 'data')
        if not os.path.isfile(launcher_jar):
            logging.warning('Cannot find {0} to determine Ultimate version'.
                            format(_LAUNCHER_JAR))
            return ''

        try:
            process = subprocess.Popen(["java", "-jar", launcher_jar, "-data", "@noDefault", "-ultimatedata", data_dir, "--version"],
                                       stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdout, stderr) = process.communicate()
        except OSError as e:
            logging.warning('Cannot run Java to determine Ultimate version: {0}'.
                            format(e.strerror))
            return ''
        if stderr:
            logging.warning('Cannot determine Ultimate version, error output: {0}'.
                            format(util.decode_to_string(stderr)))
            return ''
        if process.returncode:
            logging.warning('Cannot determine Ultimate version, exit code {0}'.
                            format(process.returncode))
            return ''

        version_ultimate_match = _ULTIMATE_VERSION_REGEX.search(util.decode_to_string(stdout))
        if not version_ultimate_match:
            logging.warning('Cannot determine Ultimate version, output: {0}'.
                            format(util.decode_to_string(stdout)))
            return ''

        return version_ultimate_match.group(1)

    @functools.lru_cache()
    def version(self, executable):
        wrapper_version = self._version_from_tool(executable)
        if wrapper_version in _SVCOMP17_VERSIONS:
            # Keep reported version number for old versions as they were before
            return wrapper_version

        ultimate_version = self._ultimate_version(executable)
        return ultimate_version + '-' + wrapper_version

    def _is_svcomp17_version(self, executable):
        return self.version(executable) in _SVCOMP17_VERSIONS

    def cmdline(self, executable, options, tasks, propertyfile=None, rlimits=None):
        if rlimits is None:
            rlimits = {}

        self._uses_propertyfile = False
        if self._is_svcomp17_version(executable):
            assert propertyfile
            self._uses_propertyfile = True
            cmdline = [executable, propertyfile]

            cmdline += [option for option in options if option not in _SVCOMP17_FORBIDDEN_FLAGS]

            cmdline.append("--full-output")

            cmdline += tasks
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using SVCOMP17 mode: ")
            return cmdline

        if propertyfile:
            # use the old wrapper script if a property file is given
            self._uses_propertyfile = True
            cmdline = [executable, '--spec', propertyfile]
            if tasks:
                cmdline += ['--file'] + tasks
            cmdline += options
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using default SVCOMP mode: ")
            return cmdline

        # if no property file is given and toolchain (-tc) is, use ultimate directly
        if '-tc' in options or '--toolchain' in options:
            # ignore executable (old executable is just around for backwards compatibility)
            mem_bytes = rlimits.get(MEMLIMIT, None)
            cmdline = ['java']
            if mem_bytes:
                cmdline += ['-Xmx' + str(mem_bytes)]

            ultimatedir = os.path.dirname(executable)
            launcher = os.path.join(ultimatedir, _LAUNCHER_JAR)
            cmdline += ['-jar', launcher]

            if '-ultimatedata' not in options:
                cmdline += ['-data', '@noDefault', '-ultimatedata', os.path.join(ultimatedir, 'data')]

            cmdline += options

            if tasks:
                cmdline += ['-i'] + tasks
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using Ultimate raw mode: ")
            return cmdline

        # there is no way to run ultimate; not enough parameters
        raise NameError("Unsupported argument combination")

    def __assert_cmdline(self, cmdline, msg):
        assert all(cmdline), msg + str(cmdline)
        pass

    def program_files(self, executable):
        install_dir = os.path.dirname(executable)
        paths = self.REQUIRED_PATHS_SVCOMP17 if self._is_svcomp17_version(executable) else self.REQUIRED_PATHS
        return [executable] + util.flatten(util.expand_filename_pattern(path, install_dir) for path in paths)

    def determine_result(self, returncode, returnsignal, output, is_timeout):
        if self._uses_propertyfile:
            return self._determine_result_with_propertyfile(returncode, returnsignal, output, is_timeout)
        return self._determine_result_without_propertyfile(returncode, returnsignal, output, is_timeout)

    def _determine_result_without_propertyfile(self, returncode, returnsignal, output, is_timeout):
        # special strings in ultimate output
        unsupported_syntax_errorstring = 'ShortDescription: Unsupported Syntax'
        incorrect_syntax_errorstring = 'ShortDescription: Incorrect Syntax'
        type_errorstring = 'Type Error'
        witness_errorstring = 'InvalidWitnessErrorResult'
        exception_errorstring = 'ExceptionOrErrorResult'
        safety_string = 'Ultimate proved your program to be correct'
        all_spec_string = 'AllSpecificationsHoldResult'
        unsafety_string = 'Ultimate proved your program to be incorrect'
        mem_deref_false_string = 'pointer dereference may fail'
        mem_deref_false_string_2 = 'array index can be out of bounds'
        mem_free_false_string = 'free of unallocated memory possible'
        mem_memtrack_false_string = 'not all allocated memory was freed'
        termination_false_string = 'Found a nonterminating execution for the following ' \
                                   'lasso shaped sequence of statements'
        termination_true_string = 'TerminationAnalysisResult: Termination proven'
        ltl_false_string = 'execution that violates the LTL property'
        ltl_true_string = 'Buchi Automizer proved that the LTL property'
        overflow_false_string = 'overflow possible'

        for line in output:
            if line.find(unsupported_syntax_errorstring) != -1:
                return 'ERROR: UNSUPPORTED SYNTAX'
            if line.find(incorrect_syntax_errorstring) != -1:
                return 'ERROR: INCORRECT SYNTAX'
            if line.find(type_errorstring) != -1:
                return 'ERROR: TYPE ERROR'
            if line.find(witness_errorstring) != -1:
                return 'ERROR: INVALID WITNESS FILE'
            if line.find(exception_errorstring) != -1:
                return 'ERROR: EXCEPTION'
            if self._contains_overapproximation_result(line):
                return 'UNKNOWN: OverapproxCex'
            if line.find(termination_true_string) != -1:
                return 'TRUE'
            if line.find(termination_false_string) != -1:
                return 'FALSE(TERM)'
            if line.find(ltl_false_string) != -1:
                return 'FALSE(valid-ltl)'
            if line.find(ltl_true_string) != -1:
                return 'TRUE'
            if line.find(safety_string) != -1 or line.find(all_spec_string) != -1:
                return 'TRUE'
            if line.find(unsafety_string) != -1:
                return 'FALSE'
            if line.find(mem_deref_false_string) != -1:
                return 'FALSE(valid-deref)'
            if line.find(mem_deref_false_string_2) != -1:
                return 'FALSE(valid-deref)'
            if line.find(mem_free_false_string) != -1:
                return 'FALSE(valid-free)'
            if line.find(mem_memtrack_false_string) != -1:
                return 'FALSE(valid-memtrack)'
            if line.find(overflow_false_string) != -1:
                return 'FALSE(OVERFLOW)'

        return result.RESULT_UNKNOWN

    def _contains_overapproximation_result(self, line):
        triggers = [
            'Reason: overapproximation of',
            'Reason: overapproximation of bitwiseAnd',
            'Reason: overapproximation of bitwiseOr',
            'Reason: overapproximation of bitwiseXor',
            'Reason: overapproximation of shiftLeft',
            'Reason: overapproximation of shiftRight',
            'Reason: overapproximation of bitwiseComplement'
        ]

        for trigger in triggers:
            if line.find(trigger) != -1:
                return True

        return False

    def _determine_result_with_propertyfile(self, returncode, returnsignal, output, is_timeout):
        for line in output:
            if line.startswith('FALSE(valid-free)'):
                return result.RESULT_FALSE_FREE
            elif line.startswith('FALSE(valid-deref)'):
                return result.RESULT_FALSE_DEREF
            elif line.startswith('FALSE(valid-memtrack)'):
                return result.RESULT_FALSE_MEMTRACK
            elif line.startswith('FALSE(TERM)'):
                return result.RESULT_FALSE_TERMINATION
            elif line.startswith('FALSE(OVERFLOW)'):
                return result.RESULT_FALSE_OVERFLOW
            elif line.startswith('FALSE'):
                return result.RESULT_FALSE_REACH
            elif line.startswith('TRUE'):
                return result.RESULT_TRUE_PROP
            elif line.startswith('UNKNOWN'):
                return result.RESULT_UNKNOWN
            elif line.startswith('ERROR'):
                status = result.RESULT_ERROR
                if line.startswith('ERROR: INVALID WITNESS FILE'):
                    status += ' (invalid witness file)'
                return status
        return result.RESULT_UNKNOWN

    def get_value_from_output(self, lines, identifier):
        # search for the text in output and get its value,
        # stop after the first line, that contains the searched text
        for line in lines:
            if identifier in line:
                start_position = line.find('=') + 1
                return line[start_position:].strip()
        return None
<commit_msg>allow older Ultimate versions that do not support ultimatedata<commit_after>"""
BenchExec is a framework for reliable benchmarking.
This file is part of BenchExec.

Copyright (C) 2015  Daniel Dietsch
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

import functools
import logging
import os
import re
import subprocess

import benchexec.result as result
import benchexec.tools.template
import benchexec.util as util
from benchexec.model import MEMLIMIT

_SVCOMP17_VERSIONS = {"f7c3ed31"}
_SVCOMP17_FORBIDDEN_FLAGS = {"--full-output", "--architecture"}
_ULTIMATE_VERSION_REGEX = re.compile('^Version is (.*)$', re.MULTILINE)
# .jar files that are used as launcher arguments with most recent .jar first
_LAUNCHER_JARS = ["plugins/org.eclipse.equinox.launcher_1.3.100.v20150511-1540.jar"]

class UltimateTool(benchexec.tools.template.BaseTool):
    """
    Abstract tool info for Ultimate-based tools.
    """

    REQUIRED_PATHS = [
        "artifacts.xml",
        "config",
        "configuration",
        "cvc4",
        "cvc4-LICENSE",
        "features",
        "LICENSE",
        "LICENSE.GPL",
        "LICENSE.GPL.LESSER",
        "mathsat",
        "p2",
        "plugins",
        "README",
        "Ultimate",
        "Ultimate.ini",
        "Ultimate.py",
        "z3",
        "z3-LICENSE"
    ]

    REQUIRED_PATHS_SVCOMP17 = []

    def __init__(self):
        self._uses_propertyfile = False

    def executable(self):
        return util.find_executable('Ultimate.py')

    def _ultimate_version(self, executable):
        data_dir = os.path.join(os.path.dirname(executable), 'data')
        launcher_jar = self._get_current_launcher_jar(executable)

        cmds = [
            ["java", "-jar", launcher_jar, "-data", "@noDefault", "-ultimatedata", data_dir, "--version"],
            ["java", "-jar", launcher_jar, "-data", data_dir, "--version"],
        ]

        api = len(cmds)
        for cmd in cmds:
            version = self._query_ultimate_version(cmd, api)
            if version != '':
                return version
            api = api - 1
        return ''

    def _query_ultimate_version(self, cmd, api):
        try:
            process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            (stdout, stderr) = process.communicate()
        except OSError as e:
            logging.warning('Cannot run Java to determine Ultimate version (API {0}): {1}'.
                            api, format(e.strerror))
            return ''
        if stderr:
            logging.warning('Cannot determine Ultimate version, error output (API {0}): {1}'.
                            api, format(util.decode_to_string(stderr)))
            return ''
        if process.returncode:
            logging.warning('Cannot determine Ultimate version, exit code (API {0}): {1}'.
                            api, format(process.returncode))
            return ''

        version_ultimate_match = _ULTIMATE_VERSION_REGEX.search(util.decode_to_string(stdout))
        if not version_ultimate_match:
            logging.warning('Cannot determine Ultimate version, output (API {0}): {1}'.
                            api, format(util.decode_to_string(stdout)))
            return ''
        return version_ultimate_match.group(1)

    @functools.lru_cache()
    def _get_current_launcher_jar(self, executable):
        ultimatedir = os.path.dirname(executable)
        for jar in _LAUNCHER_JARS:
            launcher_jar = os.path.join(ultimatedir, jar)
            if os.path.isfile(launcher_jar):
                return launcher_jar
        raise FileNotFoundError('No suitable launcher jar found')

    @functools.lru_cache()
    def version(self, executable):
        wrapper_version = self._version_from_tool(executable)
        if wrapper_version in _SVCOMP17_VERSIONS:
            # Keep reported version number for old versions as they were before
            return wrapper_version

        ultimate_version = self._ultimate_version(executable)
        return ultimate_version + '-' + wrapper_version

    @functools.lru_cache()
    def _is_svcomp17_version(self, executable):
        return self.version(executable) in _SVCOMP17_VERSIONS

    @functools.lru_cache()
    def _requires_ultimate_data(self, executable):
        if self._is_svcomp17_version(executable):
            return False;

        version = self.version(executable)
        ult, wrapper = version.split("-")
        major, minor, patch = ult.split(".")
        return not (major == 0 and minor < 2 and patch < 24)

    def cmdline(self, executable, options, tasks, propertyfile=None, rlimits=None):
        if rlimits is None:
            rlimits = {}

        self._uses_propertyfile = False
        if self._is_svcomp17_version(executable):
            assert propertyfile
            self._uses_propertyfile = True
            cmdline = [executable, propertyfile]

            cmdline += [option for option in options if option not in _SVCOMP17_FORBIDDEN_FLAGS]

            cmdline.append("--full-output")

            cmdline += tasks
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using SVCOMP17 mode: ")
            return cmdline

        if propertyfile:
            # use the old wrapper script if a property file is given
            self._uses_propertyfile = True
            cmdline = [executable, '--spec', propertyfile]
            if tasks:
                cmdline += ['--file'] + tasks
            cmdline += options
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using default SVCOMP mode: ")
            return cmdline

        # if no property file is given and toolchain (-tc) is, use ultimate directly
        if '-tc' in options or '--toolchain' in options:
            # ignore executable (old executable is just around for backwards compatibility)
            mem_bytes = rlimits.get(MEMLIMIT, None)
            cmdline = ['java']
            if mem_bytes:
                cmdline += ['-Xmx' + str(mem_bytes)]

            cmdline += ['-jar', self._get_current_launcher_jar(executable)]

            if self._requires_ultimate_data(executable):
                if '-ultimatedata' not in options and '-data' not in options:
                    cmdline += ['-data', '@noDefault', '-ultimatedata',
                                os.path.join(os.path.dirname(executable), 'data')]
                elif '-ultimatedata' in options and '-data' not in options:
                    cmdline += ['-data', '@noDefault']
            else:
                if '-data' not in options:
                    cmdline += ['-data', os.path.join(os.path.dirname(executable), 'data')]

            cmdline += options

            if tasks:
                cmdline += ['-i'] + tasks
            self.__assert_cmdline(cmdline, "cmdline contains empty or None argument when using Ultimate raw mode: ")
            return cmdline

        # there is no way to run ultimate; not enough parameters
        raise NameError("Unsupported argument combination")

    def __assert_cmdline(self, cmdline, msg):
        assert all(cmdline), msg + str(cmdline)
        pass

    def program_files(self, executable):
        install_dir = os.path.dirname(executable)
        paths = self.REQUIRED_PATHS_SVCOMP17 if self._is_svcomp17_version(executable) else self.REQUIRED_PATHS
        return [executable] + util.flatten(util.expand_filename_pattern(path, install_dir) for path in paths)

    def determine_result(self, returncode, returnsignal, output, is_timeout):
        if self._uses_propertyfile:
            return self._determine_result_with_propertyfile(returncode, returnsignal, output, is_timeout)
        return self._determine_result_without_propertyfile(returncode, returnsignal, output, is_timeout)

    def _determine_result_without_propertyfile(self, returncode, returnsignal, output, is_timeout):
        # special strings in ultimate output
        unsupported_syntax_errorstring = 'ShortDescription: Unsupported Syntax'
        incorrect_syntax_errorstring = 'ShortDescription: Incorrect Syntax'
        type_errorstring = 'Type Error'
        witness_errorstring = 'InvalidWitnessErrorResult'
        exception_errorstring = 'ExceptionOrErrorResult'
        safety_string = 'Ultimate proved your program to be correct'
        all_spec_string = 'AllSpecificationsHoldResult'
        unsafety_string = 'Ultimate proved your program to be incorrect'
        mem_deref_false_string = 'pointer dereference may fail'
        mem_deref_false_string_2 = 'array index can be out of bounds'
        mem_free_false_string = 'free of unallocated memory possible'
        mem_memtrack_false_string = 'not all allocated memory was freed'
        termination_false_string = 'Found a nonterminating execution for the following ' \
                                   'lasso shaped sequence of statements'
        termination_true_string = 'TerminationAnalysisResult: Termination proven'
        ltl_false_string = 'execution that violates the LTL property'
        ltl_true_string = 'Buchi Automizer proved that the LTL property'
        overflow_false_string = 'overflow possible'

        for line in output:
            if line.find(unsupported_syntax_errorstring) != -1:
                return 'ERROR: UNSUPPORTED SYNTAX'
            if line.find(incorrect_syntax_errorstring) != -1:
                return 'ERROR: INCORRECT SYNTAX'
            if line.find(type_errorstring) != -1:
                return 'ERROR: TYPE ERROR'
            if line.find(witness_errorstring) != -1:
                return 'ERROR: INVALID WITNESS FILE'
            if line.find(exception_errorstring) != -1:
                return 'ERROR: EXCEPTION'
            if self._contains_overapproximation_result(line):
                return 'UNKNOWN: OverapproxCex'
            if line.find(termination_true_string) != -1:
                return 'TRUE'
            if line.find(termination_false_string) != -1:
                return 'FALSE(TERM)'
            if line.find(ltl_false_string) != -1:
                return 'FALSE(valid-ltl)'
            if line.find(ltl_true_string) != -1:
                return 'TRUE'
            if line.find(safety_string) != -1 or line.find(all_spec_string) != -1:
                return 'TRUE'
            if line.find(unsafety_string) != -1:
                return 'FALSE'
            if line.find(mem_deref_false_string) != -1:
                return 'FALSE(valid-deref)'
            if line.find(mem_deref_false_string_2) != -1:
                return 'FALSE(valid-deref)'
            if line.find(mem_free_false_string) != -1:
                return 'FALSE(valid-free)'
            if line.find(mem_memtrack_false_string) != -1:
                return 'FALSE(valid-memtrack)'
            if line.find(overflow_false_string) != -1:
                return 'FALSE(OVERFLOW)'

        return result.RESULT_UNKNOWN

    def _contains_overapproximation_result(self, line):
        triggers = [
            'Reason: overapproximation of',
            'Reason: overapproximation of bitwiseAnd',
            'Reason: overapproximation of bitwiseOr',
            'Reason: overapproximation of bitwiseXor',
            'Reason: overapproximation of shiftLeft',
            'Reason: overapproximation of shiftRight',
            'Reason: overapproximation of bitwiseComplement'
        ]

        for trigger in triggers:
            if line.find(trigger) != -1:
                return True

        return False

    def _determine_result_with_propertyfile(self, returncode, returnsignal, output, is_timeout):
        for line in output:
            if line.startswith('FALSE(valid-free)'):
                return result.RESULT_FALSE_FREE
            elif line.startswith('FALSE(valid-deref)'):
                return result.RESULT_FALSE_DEREF
            elif line.startswith('FALSE(valid-memtrack)'):
                return result.RESULT_FALSE_MEMTRACK
            elif line.startswith('FALSE(TERM)'):
                return result.RESULT_FALSE_TERMINATION
            elif line.startswith('FALSE(OVERFLOW)'):
                return result.RESULT_FALSE_OVERFLOW
            elif line.startswith('FALSE'):
                return result.RESULT_FALSE_REACH
            elif line.startswith('TRUE'):
                return result.RESULT_TRUE_PROP
            elif line.startswith('UNKNOWN'):
                return result.RESULT_UNKNOWN
            elif line.startswith('ERROR'):
                status = result.RESULT_ERROR
                if line.startswith('ERROR: INVALID WITNESS FILE'):
                    status += ' (invalid witness file)'
                return status
        return result.RESULT_UNKNOWN

    def get_value_from_output(self, lines, identifier):
        # search for the text in output and get its value,
        # stop after the first line, that contains the searched text
        for line in lines:
            if identifier in line:
                start_position = line.find('=') + 1
                return line[start_position:].strip()
        return None
<|endoftext|>


<commit_before>import sys
import urllib
import urllib2
import xlwt
from bs4 import BeautifulSoup

date = sys.argv[1]
begin = sys.argv[2]
end = sys.argv[3]

wb = xlwt.Workbook()
ws = wb.add_sheet('a test sheet')

url = "http://" + date + ".stock.inv.org.cn/quote/history.php"
values = {
        "type": "daily",
        "begin_day": begin,
        "end_day": end
        }

data = urllib.urlencode(values)

headers={'User-Agent': "Mozilla/5.0"}

req = urllib2.Request(url, data, headers)

response = urllib2.urlopen(req)

page = response.read()

soup = BeautifulSoup(page)
table = soup.table

rows = table.findAll("tr")
x = 0
for tr in rows:
    cols = tr.findAll("td")
    if not cols:
        continue
    y = 0
    for td in cols:
        text_bu = td.text
        text_bu = text_bu.encode("UTF-8")
        text_bu = text_bu.strip()
        ws.write(x, y, td.text)
        #print(x, y, td.text)
        y = y + 1

    x = x + 1

filename = date + "_" + begin + "_" + end +".xls"
wb.save(filename)
<commit_msg>update variable name<commit_after>import sys
import urllib
import urllib2
import xlwt
from bs4 import BeautifulSoup

stock = sys.argv[1]
begin = sys.argv[2]
end = sys.argv[3]

wb = xlwt.Workbook()
ws = wb.add_sheet('a test sheet')

url = "http://" + stock + ".stock.inv.org.cn/quote/history.php"
values = {
        "type": "daily",
        "begin_day": begin,
        "end_day": end
        }

data = urllib.urlencode(values)

headers={'User-Agent': "Mozilla/5.0"}

req = urllib2.Request(url, data, headers)

response = urllib2.urlopen(req)

page = response.read()

soup = BeautifulSoup(page)
table = soup.table

rows = table.findAll("tr")
x = 0
for tr in rows:
    cols = tr.findAll("td")
    if not cols:
        continue
    y = 0
    for td in cols:
        text_bu = td.text
        text_bu = text_bu.encode("UTF-8")
        text_bu = text_bu.strip()
        ws.write(x, y, td.text)
        #print(x, y, td.text)
        y = y + 1

    x = x + 1

filename = stock + "_" + begin + "_" + end +".xls"
wb.save(filename)
<|endoftext|>