<commit_before>#!/usr/bin/env python

from __future__ import print_function
import os
import os.path
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from pipes import quote
except ImportError:
    from shlex import quote
import sys
import re
import subprocess
from functools import partial
import argparse
import platform

def myprint(s):
    '''
    Overriding print...
    why!? because it seems print in python3 does magic that doesn't work when we pipe through less
    '''
    try:
        if sys.version_info[0] == 3 and not hasattr(sys.stdout, 'buffer'):
            # In viewinless mode and python3... print as binary
            sys.stdout.write('{}\n'.format(s).encode('utf-8'))
        else:
            print(s)
        sys.stdout.flush()
    except IOError:
        pass

# Open dev null once
DEVNULL = open('/dev/null', 'w')

default_config_file = """; mog config file
[settings]
showname=yes
showsection=no
viewinless=no
toponly=no
toplines=10
followsymlinks=no
recursive=no

[pygments]
; Pygmentize returns 'text' when it can't highlight, so we use an inverted match for text.
invert_match=yes
pygmentize=text
arg=pygmentize

[pygmentsmime]
pygmentsmime=(.*)
argreplace=pygmentize -l '%0' %F

[elfs]
file=.*ELF
arg=objdump -ft

[filesystem]
file_mime=.*\sinode\/
arg=ls -lh{}

[media]
file_mime=.*\s(video|image)\/
arg=mediainfo

[pdfs]
file=.*PDF document
argreplace=pdftotext %F -

[tarballs]
name=.*\.tar(\.gz|\.bz2|\.Z|\.xz)?$
arg=tar --list -f

[deb]
name=.*\.deb$
arg=dpkg -I

[rpm]
name=.*\.rpm$
arg=rpm -qip

[csv]
name=.*\.csv$
arg=column -xt -s,

[tsv]
name=.*\.tsv$
arg=column -xt

[ASCII]
file=.*ASCII
arg=cat

[binary]
; We assume anything thats left is binary
name=.*
arg=xxd
""".format(" --color=always" if platform.system() != "Darwin" else "")

##### Matches
def match_file(regex, name):
    tosearch = subprocess.check_output(['file', '-h', name]).decode("utf-8")
    return re.match(regex, tosearch)

def match_file_mime(regex, name):
    tosearch = subprocess.check_output(['file', '--mime', '-k', '-h', name]).decode("utf-8").split('\n')[0]
    return re.match(regex, tosearch)

def match_pygmentize(regex, name):
    tosearch = subprocess.check_output(['pygmentize', '-N', name]).decode("utf-8")
    return re.match(regex, tosearch)

def match_pygmentsmime(regex, name):
    mimetype = subprocess.check_output(['file', '-b', '-k', '-h', '--mime-type', name]).decode("utf-8").split('\n')[0].strip()
    import pygments.lexers
    try:
        tosearch = pygments.lexers.get_lexer_for_mimetype(mimetype).aliases[0]
        if tosearch == 'text':
            return None # Skip raw text
        return re.match(regex, tosearch)
    except:
        return None

def match_inverted(func, regex, name):
    return not func(regex, name)

##### Actions
def run_program(cmd):
    try:
        subprocess.check_call(cmd, shell=True, stdout=sys.stdout, stderr=DEVNULL)
    except subprocess.CalledProcessError as e:
        myprint('==> Error processing file: {} <=='.format(e))

def action_arg(action, name, match_result, suffix):
    run_program('{} {} {}'.format(action, quote(name), suffix).strip())

def action_argreplace(action, name, match_result, suffix):
    for i, val in enumerate(match_result.groups()):
        action = action.replace('%' + str(i), val)
    run_program(action.replace('%F', quote(name) + ' ' + suffix).strip())

##### Helpers
def config_get(func, value, default, section='settings'):
    try:
        return func(section, value)
    except configparser.NoSectionError:
        return default
    except configparser.NoOptionError:
        return default
    except ValueError:
        myprint("Invalid settings variable {} in section {} - should be boolean".format(value, section))
        sys.exit(1)

def create_default_config(path):
    with open(path, 'w') as cfg:
        cfg.write(default_config_file)

##### Parsing
def parse_config(cfg):
    matches = {'file': match_file,
               'file_mime': match_file_mime,
               'name': re.match,
               'pygmentize': match_pygmentize,
               'pygmentsmime': match_pygmentsmime}
    actions = {'arg': action_arg,
               'argreplace': action_argreplace}

    # Parse config
    things = configparser.RawConfigParser()
    if not os.path.exists(cfg):
        create_default_config(cfg)
    things.read(cfg)

    # Extract settings
    settings = {'showname'   : config_get(things.getboolean, 'showname', True),
                'showsection': config_get(things.getboolean, 'showsection', False),
                'viewinless' : config_get(things.getboolean, 'viewinless', False),
                'toponly'    : config_get(things.getboolean, 'toponly', False),
                'toplines'   : config_get(things.getint, 'toplines', 10),
                'followsymlinks' : config_get(things.getboolean, 'followsymlinks', False),
                'recursive'  : config_get(things.getboolean, 'recursive', False)}

    # Extract matches and actions
    things_to_do = []
    for thing in things.sections():
        # Skip settings section
        if thing == 'settings':
            continue
        # Parse others
        invert_match = config_get(things.getboolean, 'invert_match', False, thing)
        bits = things.items(thing)
        match = None
        action = None
        for bit in bits:
            if bit[0] == 'invert_match':
                pass # Handled earlier
            elif not match and bit[0] in matches.keys():
                if invert_match:
                    match = partial(match_inverted, matches[bit[0]], bit[1])
                else:
                    match = partial(matches[bit[0]], bit[1])
            elif not action and bit[0] in actions.keys():
                action = partial(actions[bit[0]], bit[1])
            else:
                myprint("Invalid config variable {} in section {}".format(bit[0], thing))
                sys.exit(1)
        if match and action:
            things_to_do.append((match, action, thing))

    if len(things_to_do) == 0:
        myprint("Please define what you want me to do in " + cfg)
    return (settings,things_to_do)

##### Running
def run_match_action(settings, things_to_do, file_name):
    suffix = ''
    if settings['toponly']:
        suffix = '| head -n {}'.format(settings['toplines'])
    for match, action, cfg_section in things_to_do:
        match_result = match(file_name)
        if match_result:
            if settings['showname']:
                msg = file_name
                if settings['showsection']:
                    msg = "{} [{}]".format(msg, cfg_section)
                myprint('==> {} <=='.format(msg))
            action(file_name, match_result, suffix)
            return
    myprint("==> Warning: don't know what to do with {} <==".format(file_name))

def run(settings, things_to_do, files):
    first = True
    for f in files:
        if first:
            first = False
        else:
            myprint('')
        try:
            run_match_action(settings, things_to_do, f)
        except Exception as e:
            myprint('==> Error: "{}" when processing file {} <=='.format(repr(e), f))

def exists_file(f):
    if os.path.lexists(f):
        return f
    else:
        raise argparse.ArgumentTypeError("can't open {}: does not exist".format(f))

def add_pre_args(parser):
    default = os.path.expanduser("~/." + os.path.basename(sys.argv[0]) + "rc")
    parser.add_argument('-c', '--config', help='config file to use, default: {}'.format(default), default=default)

def parse_pre_args():
    parser = argparse.ArgumentParser(add_help=False)
    add_pre_args(parser)
    args, _ = parser.parse_known_args()
    return args.config

def parse_args(settings):
    help_text = {'name':    {True: 'show file names before displaying file',
                             False: 'do not show file names before displaying file'},
                 'section': {True: 'show config file section that matched before displaying file',
                             False: 'do not show config file section that matched before displaying file'},
                 'less':    {True: 'send output to less',
                             False: 'do not send output to less'},
                 'top':     {True: lambda x: 'show only top n lines of file, default (just -t): {}'.format(x),
                             False: lambda x: 'show whole files instead of only top {} lines'.format(x)},
                 'follow':  {True: 'follow symlinks to display the linked file',
                             False: 'display symlinks as is'},
                 'recurse': {True: 'recurse into directories to find files to display',
                             False: 'display directories as is'}}

    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--name', action='store_true',
            help=help_text['name'][not settings['showname']])
    parser.add_argument('-s', '--section', action='store_true',
            help=help_text['section'][not settings['showsection']])
    parser.add_argument('-l', '--less', action='store_true',
            help=help_text['less'][not settings['viewinless']])
    parser.add_argument('-t', '--top', nargs='?', const=settings['toplines'],
            help=help_text['top'][not settings['toponly']](settings['toplines']))
    parser.add_argument('-f', '--followsymlinks', action='store_true',
            help=help_text['follow'][not settings['followsymlinks']])
    parser.add_argument('-r', '--recursive', action='store_true',
            help=help_text['recurse'][not settings['recursive']])
    parser.add_argument('FILE', nargs='+', help='file(s) to process', type=exists_file)
    add_pre_args(parser)
    args = parser.parse_args()

    if args.name:
        settings['showname'] = not settings['showname']
    if args.section:
        settings['showsection'] = not settings['showsection']
    if args.less:
        settings['viewinless'] = not settings['viewinless']
    if args.top:
        settings['toponly'] = not settings['toponly']
        settings['toplines'] = args.top
    if args.followsymlinks:
        settings['followsymlinks'] = not settings['followsymlinks']
    if args.recursive:
        settings['recursive'] = not settings['recursive']

    return args.FILE

def munge_files(files, settings):
    # Note we use a set to remove duplicates when playing with symlinks
    if settings['followsymlinks']:
        files = set(filter(os.path.exists, map(os.path.realpath, files)))
    if settings['recursive']:
        newfiles = set()
        for f in files:
            if os.path.isdir(f):
                for root, dirs, newfs in os.walk(f, followlinks=settings['followsymlinks']):
                    if not settings['followsymlinks']:
                        # symlinks to dirs appear in dirs, if we aren't following them we had better add them
                        for d in dirs:
                            d = os.path.join(root, d)
                            if os.path.islink(d):
                                newfiles.add(d)
                    for newf in newfs:
                        newfile = os.path.join(root, newf)
                        if settings['followsymlinks']:
                            newfile = os.path.realpath(newfile)
                        if os.path.exists(newfile):
                            newfiles.add(newfile)
            else:
                newfiles.add(f)
        files = newfiles
    return files

def main():
    cfg = parse_pre_args()
    settings, config = parse_config(cfg)
    files = parse_args(settings)
    files = munge_files(files, settings)
    if len(config) == 0:
        sys.exit(1)
    if settings['viewinless']:
        less = subprocess.Popen(['less', '-Sr'], stdin=subprocess.PIPE)
        sys.stdout.close()
        sys.stdout = less.stdin
    else:
        less = None
    run(settings, config, files)
    try:
        sys.stdout.close()
    except BrokenPipeError:
        pass
    if less:
        less.wait()

if __name__ == "__main__":
    main()
<commit_msg>Updated filesystem regex to support version of file on osx travis builds<commit_after>#!/usr/bin/env python

from __future__ import print_function
import os
import os.path
try:
    import ConfigParser as configparser
except ImportError:
    import configparser
try:
    from pipes import quote
except ImportError:
    from shlex import quote
import sys
import re
import subprocess
from functools import partial
import argparse
import platform

def myprint(s):
    '''
    Overriding print...
    why!? because it seems print in python3 does magic that doesn't work when we pipe through less
    '''
    try:
        if sys.version_info[0] == 3 and not hasattr(sys.stdout, 'buffer'):
            # In viewinless mode and python3... print as binary
            sys.stdout.write('{}\n'.format(s).encode('utf-8'))
        else:
            print(s)
        sys.stdout.flush()
    except IOError:
        pass

# Open dev null once
DEVNULL = open('/dev/null', 'w')

default_config_file = """; mog config file
[settings]
showname=yes
showsection=no
viewinless=no
toponly=no
toplines=10
followsymlinks=no
recursive=no

[pygments]
; Pygmentize returns 'text' when it can't highlight, so we use an inverted match for text.
invert_match=yes
pygmentize=text
arg=pygmentize

[pygmentsmime]
pygmentsmime=(.*)
argreplace=pygmentize -l '%0' %F

[elfs]
file=.*ELF
arg=objdump -ft

[filesystem]
file_mime=.*(\sinode\/|x-symlink|x-directory|x-fifo)
arg=ls -lh{}

[media]
file_mime=.*\s(video|image)\/
arg=mediainfo

[pdfs]
file=.*PDF document
argreplace=pdftotext %F -

[tarballs]
name=.*\.tar(\.gz|\.bz2|\.Z|\.xz)?$
arg=tar --list -f

[deb]
name=.*\.deb$
arg=dpkg -I

[rpm]
name=.*\.rpm$
arg=rpm -qip

[csv]
name=.*\.csv$
arg=column -xt -s,

[tsv]
name=.*\.tsv$
arg=column -xt

[ASCII]
file=.*ASCII
arg=cat

[binary]
; We assume anything thats left is binary
name=.*
arg=xxd
""".format(" --color=always" if platform.system() != "Darwin" else "")

##### Matches
def match_file(regex, name):
    tosearch = subprocess.check_output(['file', '-h', name]).decode("utf-8")
    return re.match(regex, tosearch)

def match_file_mime(regex, name):
    tosearch = subprocess.check_output(['file', '--mime', '-k', '-h', name]).decode("utf-8").split('\n')[0]
    return re.match(regex, tosearch)

def match_pygmentize(regex, name):
    tosearch = subprocess.check_output(['pygmentize', '-N', name]).decode("utf-8")
    return re.match(regex, tosearch)

def match_pygmentsmime(regex, name):
    mimetype = subprocess.check_output(['file', '-b', '-k', '-h', '--mime-type', name]).decode("utf-8").split('\n')[0].strip()
    import pygments.lexers
    try:
        tosearch = pygments.lexers.get_lexer_for_mimetype(mimetype).aliases[0]
        if tosearch == 'text':
            return None # Skip raw text
        return re.match(regex, tosearch)
    except:
        return None

def match_inverted(func, regex, name):
    return not func(regex, name)

##### Actions
def run_program(cmd):
    try:
        subprocess.check_call(cmd, shell=True, stdout=sys.stdout, stderr=DEVNULL)
    except subprocess.CalledProcessError as e:
        myprint('==> Error processing file: {} <=='.format(e))

def action_arg(action, name, match_result, suffix):
    run_program('{} {} {}'.format(action, quote(name), suffix).strip())

def action_argreplace(action, name, match_result, suffix):
    for i, val in enumerate(match_result.groups()):
        action = action.replace('%' + str(i), val)
    run_program(action.replace('%F', quote(name) + ' ' + suffix).strip())

##### Helpers
def config_get(func, value, default, section='settings'):
    try:
        return func(section, value)
    except configparser.NoSectionError:
        return default
    except configparser.NoOptionError:
        return default
    except ValueError:
        myprint("Invalid settings variable {} in section {} - should be boolean".format(value, section))
        sys.exit(1)

def create_default_config(path):
    with open(path, 'w') as cfg:
        cfg.write(default_config_file)

##### Parsing
def parse_config(cfg):
    matches = {'file': match_file,
               'file_mime': match_file_mime,
               'name': re.match,
               'pygmentize': match_pygmentize,
               'pygmentsmime': match_pygmentsmime}
    actions = {'arg': action_arg,
               'argreplace': action_argreplace}

    # Parse config
    things = configparser.RawConfigParser()
    if not os.path.exists(cfg):
        create_default_config(cfg)
    things.read(cfg)

    # Extract settings
    settings = {'showname'   : config_get(things.getboolean, 'showname', True),
                'showsection': config_get(things.getboolean, 'showsection', False),
                'viewinless' : config_get(things.getboolean, 'viewinless', False),
                'toponly'    : config_get(things.getboolean, 'toponly', False),
                'toplines'   : config_get(things.getint, 'toplines', 10),
                'followsymlinks' : config_get(things.getboolean, 'followsymlinks', False),
                'recursive'  : config_get(things.getboolean, 'recursive', False)}

    # Extract matches and actions
    things_to_do = []
    for thing in things.sections():
        # Skip settings section
        if thing == 'settings':
            continue
        # Parse others
        invert_match = config_get(things.getboolean, 'invert_match', False, thing)
        bits = things.items(thing)
        match = None
        action = None
        for bit in bits:
            if bit[0] == 'invert_match':
                pass # Handled earlier
            elif not match and bit[0] in matches.keys():
                if invert_match:
                    match = partial(match_inverted, matches[bit[0]], bit[1])
                else:
                    match = partial(matches[bit[0]], bit[1])
            elif not action and bit[0] in actions.keys():
                action = partial(actions[bit[0]], bit[1])
            else:
                myprint("Invalid config variable {} in section {}".format(bit[0], thing))
                sys.exit(1)
        if match and action:
            things_to_do.append((match, action, thing))

    if len(things_to_do) == 0:
        myprint("Please define what you want me to do in " + cfg)
    return (settings,things_to_do)

##### Running
def run_match_action(settings, things_to_do, file_name):
    suffix = ''
    if settings['toponly']:
        suffix = '| head -n {}'.format(settings['toplines'])
    for match, action, cfg_section in things_to_do:
        match_result = match(file_name)
        if match_result:
            if settings['showname']:
                msg = file_name
                if settings['showsection']:
                    msg = "{} [{}]".format(msg, cfg_section)
                myprint('==> {} <=='.format(msg))
            action(file_name, match_result, suffix)
            return
    myprint("==> Warning: don't know what to do with {} <==".format(file_name))

def run(settings, things_to_do, files):
    first = True
    for f in files:
        if first:
            first = False
        else:
            myprint('')
        try:
            run_match_action(settings, things_to_do, f)
        except Exception as e:
            myprint('==> Error: "{}" when processing file {} <=='.format(repr(e), f))

def exists_file(f):
    if os.path.lexists(f):
        return f
    else:
        raise argparse.ArgumentTypeError("can't open {}: does not exist".format(f))

def add_pre_args(parser):
    default = os.path.expanduser("~/." + os.path.basename(sys.argv[0]) + "rc")
    parser.add_argument('-c', '--config', help='config file to use, default: {}'.format(default), default=default)

def parse_pre_args():
    parser = argparse.ArgumentParser(add_help=False)
    add_pre_args(parser)
    args, _ = parser.parse_known_args()
    return args.config

def parse_args(settings):
    help_text = {'name':    {True: 'show file names before displaying file',
                             False: 'do not show file names before displaying file'},
                 'section': {True: 'show config file section that matched before displaying file',
                             False: 'do not show config file section that matched before displaying file'},
                 'less':    {True: 'send output to less',
                             False: 'do not send output to less'},
                 'top':     {True: lambda x: 'show only top n lines of file, default (just -t): {}'.format(x),
                             False: lambda x: 'show whole files instead of only top {} lines'.format(x)},
                 'follow':  {True: 'follow symlinks to display the linked file',
                             False: 'display symlinks as is'},
                 'recurse': {True: 'recurse into directories to find files to display',
                             False: 'display directories as is'}}

    parser = argparse.ArgumentParser()
    parser.add_argument('-n', '--name', action='store_true',
            help=help_text['name'][not settings['showname']])
    parser.add_argument('-s', '--section', action='store_true',
            help=help_text['section'][not settings['showsection']])
    parser.add_argument('-l', '--less', action='store_true',
            help=help_text['less'][not settings['viewinless']])
    parser.add_argument('-t', '--top', nargs='?', const=settings['toplines'],
            help=help_text['top'][not settings['toponly']](settings['toplines']))
    parser.add_argument('-f', '--followsymlinks', action='store_true',
            help=help_text['follow'][not settings['followsymlinks']])
    parser.add_argument('-r', '--recursive', action='store_true',
            help=help_text['recurse'][not settings['recursive']])
    parser.add_argument('FILE', nargs='+', help='file(s) to process', type=exists_file)
    add_pre_args(parser)
    args = parser.parse_args()

    if args.name:
        settings['showname'] = not settings['showname']
    if args.section:
        settings['showsection'] = not settings['showsection']
    if args.less:
        settings['viewinless'] = not settings['viewinless']
    if args.top:
        settings['toponly'] = not settings['toponly']
        settings['toplines'] = args.top
    if args.followsymlinks:
        settings['followsymlinks'] = not settings['followsymlinks']
    if args.recursive:
        settings['recursive'] = not settings['recursive']

    return args.FILE

def munge_files(files, settings):
    # Note we use a set to remove duplicates when playing with symlinks
    if settings['followsymlinks']:
        files = set(filter(os.path.exists, map(os.path.realpath, files)))
    if settings['recursive']:
        newfiles = set()
        for f in files:
            if os.path.isdir(f):
                for root, dirs, newfs in os.walk(f, followlinks=settings['followsymlinks']):
                    if not settings['followsymlinks']:
                        # symlinks to dirs appear in dirs, if we aren't following them we had better add them
                        for d in dirs:
                            d = os.path.join(root, d)
                            if os.path.islink(d):
                                newfiles.add(d)
                    for newf in newfs:
                        newfile = os.path.join(root, newf)
                        if settings['followsymlinks']:
                            newfile = os.path.realpath(newfile)
                        if os.path.exists(newfile):
                            newfiles.add(newfile)
            else:
                newfiles.add(f)
        files = newfiles
    return files

def main():
    cfg = parse_pre_args()
    settings, config = parse_config(cfg)
    files = parse_args(settings)
    files = munge_files(files, settings)
    if len(config) == 0:
        sys.exit(1)
    if settings['viewinless']:
        less = subprocess.Popen(['less', '-Sr'], stdin=subprocess.PIPE)
        sys.stdout.close()
        sys.stdout = less.stdin
    else:
        less = None
    run(settings, config, files)
    try:
        sys.stdout.close()
    except BrokenPipeError:
        pass
    if less:
        less.wait()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
import time
import random

import pytest

from selenium.common.exceptions import WebDriverException

@pytest.fixture
def cookie(webserver):
    cookie = {
        'name': 'foo',
        'value': 'bar',
        'domain': webserver.host,
        'path': '/',
        'secure': False}
    return cookie

@pytest.fixture
def same_site_cookie_strict(webserver):
    same_site_cookie_strict = {
        'name': 'foo',
        'value': 'bar',
        'path': '/',
        'domain': webserver.host,
        'sameSite': 'Strict',
        'secure': False}
    return same_site_cookie_strict

@pytest.fixture
def same_site_cookie_lax(webserver):
    same_site_cookie_lax = {
        'name': 'foo',
        'value': 'bar',
        'path': '/',
        'domain': webserver.host,
        'sameSite': 'Lax',
        'secure': False}
    return same_site_cookie_lax

@pytest.fixture(autouse=True)
def pages(request, driver, pages):
    pages.load('simpleTest.html')
    yield pages
    driver.delete_all_cookies()

def testAddCookie(cookie, driver):
    driver.add_cookie(cookie)
    returned = driver.execute_script('return document.cookie')
    assert cookie['name'] in returned

@pytest.mark.xfail_firefox(raises=WebDriverException,
                           reason='sameSite cookie attribute not implemented')
def testAddCookieSameSiteStrict(same_site_cookie_strict, driver):
    driver.add_cookie(same_site_cookie_strict)
    returned = driver.get_cookie('foo')
    assert returned['sameSite'] == 'Strict'

@pytest.mark.xfail_firefox(raises=WebDriverException,
                           reason='sameSite cookie attribute not implemented')
def testAddCookieSameSiteLax(same_site_cookie_lax, driver):
    driver.add_cookie(same_site_cookie_lax)
    returned = driver.get_cookie('foo')
    assert returned['sameSite'] == 'Lax'

@pytest.mark.xfail_ie
def testAddingACookieThatExpiredInThePast(cookie, driver):
    expired = cookie.copy()
    expired['expiry'] = calendar.timegm(time.gmtime()) - 1
    driver.add_cookie(expired)
    assert 0 == len(driver.get_cookies())

def testDeleteAllCookie(cookie, driver):
    driver.add_cookie(cookie)
    driver.delete_all_cookies()
    assert not driver.get_cookies()

def testDeleteCookie(cookie, driver):
    driver.add_cookie(cookie)
    driver.delete_cookie('foo')
    assert not driver.get_cookies()

def testShouldGetCookieByName(driver):
    key = 'key_{}'.format(int(random.random() * 10000000))
    driver.execute_script("document.cookie = arguments[0] + '=set';", key)
    cookie = driver.get_cookie(key)
    assert 'set' == cookie['value']

def testShouldReturnNoneWhenCookieDoesNotExist(driver):
    key = 'key_{}'.format(int(random.random() * 10000000))
    cookie = driver.get_cookie(key)
    assert cookie is None

def testGetAllCookies(cookie, driver, pages, webserver):
    cookies = driver.get_cookies()
    count = len(cookies)

    for i in range(2):
        cookie['name'] = 'key_{}'.format(int(random.random() * 10000000))
        driver.add_cookie(cookie)

    pages.load('simpleTest.html')
    assert count + 2 == len(driver.get_cookies())

def testShouldNotDeleteCookiesWithASimilarName(cookie, driver, webserver):
    cookie2 = cookie.copy()
    cookie2['name'] = '{}x'.format(cookie['name'])
    driver.add_cookie(cookie)
    driver.add_cookie(cookie2)
    driver.delete_cookie(cookie['name'])
    cookies = driver.get_cookies()
    assert cookie['name'] != cookies[0]['name']
    assert cookie2['name'] == cookies[0]['name']
<commit_msg>[py] Fixing expected failure for same site cookie test in Firefox<commit_after># Licensed to the Software Freedom Conservancy (SFC) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The SFC licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

import calendar
import time
import random

import pytest

@pytest.fixture
def cookie(webserver):
    cookie = {
        'name': 'foo',
        'value': 'bar',
        'domain': webserver.host,
        'path': '/',
        'secure': False}
    return cookie

@pytest.fixture
def same_site_cookie_strict(webserver):
    same_site_cookie_strict = {
        'name': 'foo',
        'value': 'bar',
        'path': '/',
        'domain': webserver.host,
        'sameSite': 'Strict',
        'secure': False}
    return same_site_cookie_strict

@pytest.fixture
def same_site_cookie_lax(webserver):
    same_site_cookie_lax = {
        'name': 'foo',
        'value': 'bar',
        'path': '/',
        'domain': webserver.host,
        'sameSite': 'Lax',
        'secure': False}
    return same_site_cookie_lax

@pytest.fixture(autouse=True)
def pages(request, driver, pages):
    pages.load('simpleTest.html')
    yield pages
    driver.delete_all_cookies()

def testAddCookie(cookie, driver):
    driver.add_cookie(cookie)
    returned = driver.execute_script('return document.cookie')
    assert cookie['name'] in returned

@pytest.mark.xfail_firefox(reason='sameSite cookie attribute not implemented')
def testAddCookieSameSiteStrict(same_site_cookie_strict, driver):
    driver.add_cookie(same_site_cookie_strict)
    returned = driver.get_cookie('foo')
    assert 'sameSite' in returned and returned['sameSite'] == 'Strict'

@pytest.mark.xfail_firefox(reason='sameSite cookie attribute not implemented')
def testAddCookieSameSiteLax(same_site_cookie_lax, driver):
    driver.add_cookie(same_site_cookie_lax)
    returned = driver.get_cookie('foo')
    assert 'sameSite' in returned and returned['sameSite'] == 'Lax'

@pytest.mark.xfail_ie
def testAddingACookieThatExpiredInThePast(cookie, driver):
    expired = cookie.copy()
    expired['expiry'] = calendar.timegm(time.gmtime()) - 1
    driver.add_cookie(expired)
    assert 0 == len(driver.get_cookies())

def testDeleteAllCookie(cookie, driver):
    driver.add_cookie(cookie)
    driver.delete_all_cookies()
    assert not driver.get_cookies()

def testDeleteCookie(cookie, driver):
    driver.add_cookie(cookie)
    driver.delete_cookie('foo')
    assert not driver.get_cookies()

def testShouldGetCookieByName(driver):
    key = 'key_{}'.format(int(random.random() * 10000000))
    driver.execute_script("document.cookie = arguments[0] + '=set';", key)
    cookie = driver.get_cookie(key)
    assert 'set' == cookie['value']

def testShouldReturnNoneWhenCookieDoesNotExist(driver):
    key = 'key_{}'.format(int(random.random() * 10000000))
    cookie = driver.get_cookie(key)
    assert cookie is None

def testGetAllCookies(cookie, driver, pages, webserver):
    cookies = driver.get_cookies()
    count = len(cookies)

    for i in range(2):
        cookie['name'] = 'key_{}'.format(int(random.random() * 10000000))
        driver.add_cookie(cookie)

    pages.load('simpleTest.html')
    assert count + 2 == len(driver.get_cookies())

def testShouldNotDeleteCookiesWithASimilarName(cookie, driver, webserver):
    cookie2 = cookie.copy()
    cookie2['name'] = '{}x'.format(cookie['name'])
    driver.add_cookie(cookie)
    driver.add_cookie(cookie2)
    driver.delete_cookie(cookie['name'])
    cookies = driver.get_cookies()
    assert cookie['name'] != cookies[0]['name']
    assert cookie2['name'] == cookies[0]['name']
<|endoftext|>


<commit_before>def newTrie(*words):
    tree = dict() #the tree as a dictionary
    for word in words:
        currentDict = tree
        for letter in word:
            currentDict = currentDict.setdefault(letter,{})
            #currentDict['value'] = value
        if('value' in currentDict):
            currentDict['value'] = currentDict['value'] + 1
        else:
            currentDict['value'] = 1

    return tree

#Function that adds words in a already existed Trie's Tree
#It works very similar to the creation of a new Tree's
def addTrie(trie,*words):
    currentDic = trie #It  receives the Tree's to a variable that will change
    for word in words:
        currentDict = currentDic #CurrentDic will bet the variable that is going to get in the Dictonary
        for letter in word:
            if(letter in currentDict):
                currentDict = currentDict[letter] #This will enable that if the letter already exists it will just get in it, NOT CREATE A NEW
            else:
                currentDict = currentDict.setdefault(letter,{}) #And if not, it will Create one
        if('value' in currentDict):
            currentDict['value'] = currentDict['value'] + 1  #If for some reason the word already exists, it will only Add +1 to the Counter
        else:
            currentDict['value'] = 1 #If not, it will create the Counter

    return currentDic #Returns the Trie's Tree

#you need to pass the Trie Tree and the word you are looking for
def inTrie(trie, word):
    currentDict = trie
    for letter in word:
        #it runs every letter
        if(letter in currentDict):
            #it will run until the last letter found
            currentDict = currentDict[letter]
        else:
            return False
    else:
        #if it runs and FIND ALL LETTERS IN THE WORD
        #it will look if it has a value on it
        if ('value' in currentDict):
            return True
        else:
            return False

#remover
#In Progress
def remTrie(trie,*words):
    currentDic = trie
    for word in words:
        currentDict = currentDic
        for letter in word:
            if(letter in currentDict): #it will run until the last letter found
                currentDict = currentDict[letter]
        else:
            if('value' in currentDict): # the last letter of a phrase
                if(len(currentDict) == 1):
                    del[currentDict['value']]
                    #del[currentDict]
                elif(len(currentDict) > 1):
                    del[currentDict['value']]

    return currentDic

#DEBUGER - OUTPUT
arvore = newTrie('bahea','salvado','salvador','sal','sal','bahea','bahia','sergipe')
print(arvore)
arvore = addTrie(arvore,'banana','presunto','bahia')
print('\n\n\n')
print(arvore)
arvore = remTrie(arvore,'salvador','salvado')
print('\n\n\n')
print(arvore)
print(inTrie(arvore,'ser'))
print(inTrie(arvore,'presunto'))
<commit_msg>Started Get<commit_after>def newTrie(*words):
    tree = dict() #the tree as a dictionary
    for word in words:
        currentDict = tree
        for letter in word:
            currentDict = currentDict.setdefault(letter,{})
            #currentDict['value'] = value
        if('value' in currentDict):
            currentDict['value'] = currentDict['value'] + 1
        else:
            currentDict['value'] = 1

    return tree

def getValue(trie):
    currentDict = trie
    raiz = list(currentDict)
    for i in range(len(raiz)): #percorrer todos os nos

        #parei de programar pois percebi que devera ser algo recursivo e nao forçar com loops :(

#Function that adds words in a already existed Trie's Tree
#It works very similar to the creation of a new Tree's
def addTrie(trie,*words):
    currentDic = trie #It  receives the Tree's to a variable that will change
    for word in words:
        currentDict = currentDic #CurrentDic will bet the variable that is going to get in the Dictonary
        for letter in word:
            if(letter in currentDict):
                currentDict = currentDict[letter] #This will enable that if the letter already exists it will just get in it, NOT CREATE A NEW
            else:
                currentDict = currentDict.setdefault(letter,{}) #And if not, it will Create one
        if('value' in currentDict):
            currentDict['value'] = currentDict['value'] + 1  #If for some reason the word already exists, it will only Add +1 to the Counter
        else:
            currentDict['value'] = 1 #If not, it will create the Counter

    return currentDic #Returns the Trie's Tree

#you need to pass the Trie Tree and the word you are looking for
def inTrie(trie, word):
    currentDict = trie
    for letter in word:
        #it runs every letter
        if(letter in currentDict):
            #it will run until the last letter found
            currentDict = currentDict[letter]
        else:
            return False
    else:
        #if it runs and FIND ALL LETTERS IN THE WORD
        #it will look if it has a value on it
        if ('value' in currentDict):
            return True
        else:
            return False

#remover
#In Progress
def remTrie(trie,*words):
    currentDic = trie
    for word in words:
        currentDict = currentDic
        for letter in word:
            if(letter in currentDict): #it will run until the last letter found
                currentDict = currentDict[letter]
        else:
            if('value' in currentDict): # the last letter of a phrase
                if(len(currentDict) == 1):
                    del[currentDict['value']]
                    #del[currentDict]
                elif(len(currentDict) > 1):
                    del[currentDict['value']]

    return currentDic

#DEBUGER - OUTPUT
arvore = newTrie('bahea','salvado','salvador','sal','sal','bahea','bahia','sergipe')
print(arvore)
arvore = addTrie(arvore,'banana','presunto','bahia')
print('\n\n\n')
print(arvore)
arvore = remTrie(arvore,'salvador','salvado')
print('\n\n\n')
print(arvore)
print(inTrie(arvore,'ser'))
print(inTrie(arvore,'presunto'))
<|endoftext|>


<commit_before>from __future__ import division
from collections import defaultdict
import os
import dill
import numpy as np

import pySDC.helpers.plot_helper as plt_helper

from pySDC.implementations.collocation_classes.gauss_lobatto import CollGaussLobatto
from pySDC.implementations.controller_classes.controller_nonMPI import controller_nonMPI
from pySDC.implementations.sweeper_classes.verlet import verlet
from pySDC.implementations.datatype_classes.particles import particles, acceleration
from pySDC.implementations.problem_classes.HarmonicOscillator import harmonic_oscillator
from pySDC.implementations.transfer_classes.TransferParticles_NoCoarse import particles_to_particles

from pySDC.helpers.stats_helper import filter_stats, sort_stats
from pySDC.projects.Hamiltonian.hamiltonian_output import hamiltonian_output

def run_simulation():
    """
    Routine to run the simulation of a second order problem

    """

    # initialize level parameters
    level_params = dict()
    level_params['restol'] = 1E-10
    level_params['dt'] = 1.0

    # initialize sweeper parameters
    sweeper_params = dict()
    sweeper_params['collocation_class'] = CollGaussLobatto
    sweeper_params['num_nodes'] = 5
    sweeper_params['spread'] = False

    # initialize problem parameters for the Penning trap
    problem_params = dict()
    problem_params['k'] = None  # will be defined later
    problem_params['phase'] = 0.0
    problem_params['amp'] = 1.0

    # initialize step parameters
    step_params = dict()
    step_params['maxiter'] = 50

    # initialize controller parameters
    controller_params = dict()
    controller_params['hook_class'] = hamiltonian_output  # specialized hook class for more statistics and output
    controller_params['logger_level'] = 30

    # Fill description dictionary for easy hierarchy creation
    description = dict()
    description['problem_class'] = harmonic_oscillator
    description['dtype_u'] = particles
    description['dtype_f'] = acceleration
    description['sweeper_class'] = verlet
    description['level_params'] = level_params
    description['step_params'] = step_params
    # description['space_transfer_class'] = particles_to_particles

    # set time parameters
    t0 = 0.0
    Tend = 1.0
    num_procs = 1

    rlim_left = 0
    rlim_right = 50
    nstep = 26
    ks = np.linspace(rlim_left, rlim_right, nstep)[1:]

    qd_combinations = [('IE', 'EE'), ('IE', 'PIC'),
                       ('LU', 'EE'), ('LU', 'PIC'),
                       ('MIN3', 'PIC'), ('MIN3', 'EE'),
                       ('PIC', 'EE'), ('PIC', 'PIC')]

    results = dict()
    results['ks'] = ks

    for qd in qd_combinations:

        print('Working on combination (%s, %s)...' % qd)

        niters = np.zeros(len(ks))

        for i, k in enumerate(ks):

            problem_params['k'] = k
            description['problem_params'] = problem_params

            sweeper_params['QI'] = qd[0]
            sweeper_params['QE'] = qd[1]
            description['sweeper_params'] = sweeper_params

            # instantiate the controller
            controller = controller_nonMPI(num_procs=num_procs, controller_params=controller_params,
                                           description=description)

            # get initial values on finest level
            P = controller.MS[0].levels[0].prob
            uinit = P.u_exact(t=t0)

            # call main function to get things done...
            uend, stats = controller.run(u0=uinit, t0=t0, Tend=Tend)

            # filter statistics by type (number of iterations)
            filtered_stats = filter_stats(stats, type='niter')

            # convert filtered statistics to list of iterations count, sorted by process
            iter_counts = sort_stats(filtered_stats, sortby='time')

            assert len(iter_counts) == 1, 'ERROR: more than one record for iterations, got %s' % iter_counts

            niters[i] = iter_counts[0][1]

            # print('Worked on k = %s, took %s iterations' % (k, results[i]))

        results[qd] = niters

    fname = 'data/harmonic_k.dat'
    f = open(fname, 'wb')
    dill.dump(results, f)
    f.close()

    assert os.path.isfile(fname), 'Run for %s did not create stats file' % prob

def show_results(cwd=''):
    """
    Helper function to plot the error of the Hamiltonian

    Args:
        cwd (str): current working directory
    """

    plt_helper.mpl.style.use('classic')
    plt_helper.setup_mpl()
    plt_helper.newfig(textwidth=238.96, scale=0.89)

    # read in the dill data
    f = open(cwd + 'data/harmonic_k.dat', 'rb')
    results = dill.load(f)
    f.close()

    ks = results['ks']

    for qd in results:

        if qd != 'ks':

            plt_helper.plt.plot(ks, results[qd], label=qd)

    plt_helper.plt.xlabel('k')
    plt_helper.plt.ylabel('Number of iterations')
    plt_helper.plt.legend(loc='lower right')

    fname = 'data/harmonic_qd_iterations'
    plt_helper.savefig(fname)

    assert os.path.isfile(fname + '.pdf'), 'ERROR: plotting did not create PDF file'
    assert os.path.isfile(fname + '.pgf'), 'ERROR: plotting did not create PGF file'
    assert os.path.isfile(fname + '.png'), 'ERROR: plotting did not create PNG file'

def main():
    run_simulation()
    show_results()

if __name__ == "__main__":
    main()
<commit_msg>more testing<commit_after>from __future__ import division
from collections import defaultdict
import os
import dill
import numpy as np

import pySDC.helpers.plot_helper as plt_helper

from pySDC.implementations.collocation_classes.gauss_lobatto import CollGaussLobatto
from pySDC.implementations.controller_classes.controller_nonMPI import controller_nonMPI
from pySDC.implementations.sweeper_classes.verlet import verlet
from pySDC.implementations.datatype_classes.particles import particles, acceleration
from pySDC.implementations.problem_classes.HarmonicOscillator import harmonic_oscillator
from pySDC.implementations.transfer_classes.TransferParticles_NoCoarse import particles_to_particles

from pySDC.helpers.stats_helper import filter_stats, sort_stats
from pySDC.projects.Hamiltonian.hamiltonian_output import hamiltonian_output

def run_simulation():
    """
    Routine to run the simulation of a second order problem

    """

    # initialize level parameters
    level_params = dict()
    level_params['restol'] = 1E-10
    level_params['dt'] = 1.0

    # initialize sweeper parameters
    sweeper_params = dict()
    sweeper_params['collocation_class'] = CollGaussLobatto
    sweeper_params['num_nodes'] = 5
    sweeper_params['spread'] = False

    # initialize problem parameters for the Penning trap
    problem_params = dict()
    problem_params['k'] = None  # will be defined later
    problem_params['phase'] = 0.0
    problem_params['amp'] = 1.0

    # initialize step parameters
    step_params = dict()
    step_params['maxiter'] = 50

    # initialize controller parameters
    controller_params = dict()
    controller_params['hook_class'] = hamiltonian_output  # specialized hook class for more statistics and output
    controller_params['logger_level'] = 30

    # Fill description dictionary for easy hierarchy creation
    description = dict()
    description['problem_class'] = harmonic_oscillator
    description['dtype_u'] = particles
    description['dtype_f'] = acceleration
    description['sweeper_class'] = verlet
    description['level_params'] = level_params
    description['step_params'] = step_params
    # description['space_transfer_class'] = particles_to_particles

    # set time parameters
    t0 = 0.0
    Tend = 1.0
    num_procs = 1

    rlim_left = 0
    rlim_right = 50
    nstep = 26
    ks = np.linspace(rlim_left, rlim_right, nstep)[1:]

    qd_combinations = [('IE', 'EE'), ('IE', 'PIC'),
                       ('LU', 'EE'), ('LU', 'PIC'),
                       # ('MIN3', 'PIC'), ('MIN3', 'EE'),
                       ('PIC', 'EE'), ('PIC', 'PIC')]

    results = dict()
    results['ks'] = ks

    for qd in qd_combinations:

        print('Working on combination (%s, %s)...' % qd)

        niters = np.zeros(len(ks))

        for i, k in enumerate(ks):

            problem_params['k'] = k
            description['problem_params'] = problem_params

            sweeper_params['QI'] = qd[0]
            sweeper_params['QE'] = qd[1]
            description['sweeper_params'] = sweeper_params

            # instantiate the controller
            controller = controller_nonMPI(num_procs=num_procs, controller_params=controller_params,
                                           description=description)

            # get initial values on finest level
            P = controller.MS[0].levels[0].prob
            uinit = P.u_exact(t=t0)

            # call main function to get things done...
            uend, stats = controller.run(u0=uinit, t0=t0, Tend=Tend)

            # filter statistics by type (number of iterations)
            filtered_stats = filter_stats(stats, type='niter')

            # convert filtered statistics to list of iterations count, sorted by process
            iter_counts = sort_stats(filtered_stats, sortby='time')

            assert len(iter_counts) == 1, 'ERROR: more than one record for iterations, got %s' % iter_counts

            niters[i] = iter_counts[0][1]

            # print('Worked on k = %s, took %s iterations' % (k, results[i]))

        results[qd] = niters

    fname = 'data/harmonic_k.dat'
    f = open(fname, 'wb')
    dill.dump(results, f)
    f.close()

    assert os.path.isfile(fname), 'Run for %s did not create stats file' % prob

def show_results(cwd=''):
    """
    Helper function to plot the error of the Hamiltonian

    Args:
        cwd (str): current working directory
    """

    plt_helper.mpl.style.use('classic')
    plt_helper.setup_mpl()
    plt_helper.newfig(textwidth=238.96, scale=0.89)

    # read in the dill data
    f = open(cwd + 'data/harmonic_k.dat', 'rb')
    results = dill.load(f)
    f.close()

    ks = results['ks']

    for qd in results:

        if qd != 'ks':

            plt_helper.plt.plot(ks, results[qd], label=qd)

    plt_helper.plt.xlabel('k')
    plt_helper.plt.ylabel('Number of iterations')
    plt_helper.plt.legend(loc='lower right', )

    fname = 'data/harmonic_qd_iterations'
    plt_helper.savefig(fname)

    assert os.path.isfile(fname + '.pdf'), 'ERROR: plotting did not create PDF file'
    assert os.path.isfile(fname + '.pgf'), 'ERROR: plotting did not create PGF file'
    assert os.path.isfile(fname + '.png'), 'ERROR: plotting did not create PNG file'

def main():
    run_simulation()
    show_results()

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import os.path
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import sys

from trac.util import get_doc, get_module_path, get_sources, get_pkginfo
from trac.util.text import exception_to_unicode, to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module if it wasn't disabled explicitly."""
    if env.is_component_enabled(module) is None:
        env.enable_component(module)

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            if dist not in working_set:
                env.log.debug('Adding plugin %s from %s', dist, dist.location)
                working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": %s', item,
                              exception_to_unicode(e, traceback=True))

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in sorted(working_set.iter_entry_points(entry_point_name),
                            key=lambda entry: entry.name):
            env.log.debug('Loading %s from %s', entry.name, entry.dist.location)
            try:
                entry.load(require=True)
            except Exception, e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s: %s',
                                  plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def get_plugins_dir(env):
    """Return the path to the `plugins` directory of the environment."""
    plugins_dir = os.path.realpath(os.path.join(env.path, 'plugins'))
    return os.path.normcase(plugins_dir)

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = get_plugins_dir(env)
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)

def get_plugin_info(env, include_core=False):
    """Return package information about Trac core and installed plugins."""
    path_sources = {}

    def find_distribution(module):
        name = module.__name__
        path = get_module_path(module)
        sources = path_sources.get(path)
        if sources is None:
            sources = path_sources[path] = get_sources(path)
        dist = sources.get(name.replace('.', '/') + '.py')
        if dist is None:
            dist = sources.get(name.replace('.', '/') + '/__init__.py')
        if dist is None:
            # This is a plain Python source file, not an egg
            dist = pkg_resources.Distribution(project_name=name,
                                              version='',
                                              location=module.__file__)
        return dist

    plugins_dir = get_plugins_dir(env)
    plugins = {}
    from trac.core import ComponentMeta
    for component in ComponentMeta._components:
        module = sys.modules[component.__module__]

        dist = find_distribution(module)
        plugin_filename = None
        if os.path.realpath(os.path.dirname(dist.location)) == plugins_dir:
            plugin_filename = os.path.basename(dist.location)

        if dist.project_name not in plugins:
            readonly = True
            if plugin_filename and os.access(dist.location,
                                             os.F_OK + os.W_OK):
                readonly = False
            # retrieve plugin metadata
            info = get_pkginfo(dist)
            if not info:
                info = {}
                for k in ('author', 'author_email', 'home_page', 'url',
                          'license', 'trac'):
                    v = getattr(module, k, '')
                    if v and isinstance(v, basestring):
                        if k == 'home_page' or k == 'url':
                            k = 'home_page'
                            v = v.replace('$', '').replace('URL: ', '')
                        else:
                            v = to_unicode(v)
                        info[k] = v
            else:
                # Info found; set all those fields to "None" that have the
                # value "UNKNOWN" as this is the value for fields that
                # aren't specified in "setup.py"
                for k in info:
                    if info[k] == 'UNKNOWN':
                        info[k] = ''
                    else:
                        # Must be encoded as unicode as otherwise Genshi
                        # may raise a "UnicodeDecodeError".
                        info[k] = to_unicode(info[k])

            # retrieve plugin version info
            version = dist.version
            if not version:
                version = (getattr(module, 'version', '') or
                           getattr(module, 'revision', ''))
                # special handling for "$Rev$" strings
                version = version.replace('$', '').replace('Rev: ', 'r')
            plugins[dist.project_name] = {
                'name': dist.project_name, 'version': version,
                'path': dist.location, 'plugin_filename': plugin_filename,
                'readonly': readonly, 'info': info, 'modules': {},
            }
        modules = plugins[dist.project_name]['modules']
        if module.__name__ not in modules:
            summary, description = get_doc(module)
            plugins[dist.project_name]['modules'][module.__name__] = {
                'summary': summary, 'description': description,
                'components': {},
            }
        full_name = module.__name__ + '.' + component.__name__
        summary, description = get_doc(component)
        c = component
        if c in env and not issubclass(c, env.__class__):
            c = component(env)
        modules[module.__name__]['components'][component.__name__] = {
            'full_name': full_name,
            'summary': summary, 'description': description,
            'enabled': env.is_component_enabled(component),
            'required': getattr(c, 'required', False),
        }
    if not include_core:
        for name in plugins.keys():
            if name.lower() == 'trac':
                plugins.pop(name)
    return sorted(plugins.itervalues(),
                  key=lambda p: (p['name'].lower() != 'trac',
                                 p['name'].lower()))

def match_plugins_to_frames(plugins, frames):
    """Add a `frame_idx` element to plugin information as returned by
    `get_plugin_info()`, containing the index of the highest frame in the
    list that was located in the plugin.
    """
    egg_frames = [(i, f) for i, f in enumerate(frames)
                  if f['filename'].startswith('build/')]

    def find_egg_frame_index(plugin):
        for dist in pkg_resources.find_distributions(plugin['path'],
                                                     only=True):
            try:
                sources = dist.get_metadata('SOURCES.txt')
                for src in sources.splitlines():
                    if src.endswith('.py'):
                        nsrc = src.replace('\\', '/')
                        for i, f in egg_frames:
                            if f['filename'].endswith(nsrc):
                                plugin['frame_idx'] = i
                                return
            except KeyError:
                pass    # Metadata not found

    for plugin in plugins:
        base, ext = os.path.splitext(plugin['path'])
        if ext == '.egg' and egg_frames:
            find_egg_frame_index(plugin)
        else:
            for i, f in enumerate(frames):
                if f['filename'].startswith(base):
                    plugin['frame_idx'] = i
                    break
<commit_msg>0.12.6dev: fixed file-based plugin detection not working on Windows (refs #11434)<commit_after># -*- coding: utf-8 -*-
#
# Copyright (C) 2005-2009 Edgewall Software
# Copyright (C) 2005-2006 Christopher Lenz <cmlenz@gmx.de>
# All rights reserved.
#
# This software is licensed as described in the file COPYING, which
# you should have received as part of this distribution. The terms
# are also available at http://trac.edgewall.org/wiki/TracLicense.
#
# This software consists of voluntary contributions made by many
# individuals. For the exact contribution history, see the revision
# history and logs, available at http://trac.edgewall.org/log/.
#
# Author: Christopher Lenz <cmlenz@gmx.de>

from glob import glob
import imp
import os.path
import pkg_resources
from pkg_resources import working_set, DistributionNotFound, VersionConflict, \
                          UnknownExtra
import sys

from trac.util import get_doc, get_module_path, get_sources, get_pkginfo
from trac.util.text import exception_to_unicode, to_unicode

__all__ = ['load_components']

def _enable_plugin(env, module):
    """Enable the given plugin module if it wasn't disabled explicitly."""
    if env.is_component_enabled(module) is None:
        env.enable_component(module)

def load_eggs(entry_point_name):
    """Loader that loads any eggs on the search path and `sys.path`."""
    def _load_eggs(env, search_path, auto_enable=None):
        # Note that the following doesn't seem to support unicode search_path
        distributions, errors = working_set.find_plugins(
            pkg_resources.Environment(search_path)
        )
        for dist in distributions:
            if dist not in working_set:
                env.log.debug('Adding plugin %s from %s', dist, dist.location)
                working_set.add(dist)

        def _log_error(item, e):
            ue = exception_to_unicode(e)
            if isinstance(e, DistributionNotFound):
                env.log.debug('Skipping "%s": ("%s" not found)', item, ue)
            elif isinstance(e, VersionConflict):
                env.log.error('Skipping "%s": (version conflict "%s")',
                              item, ue)
            elif isinstance(e, UnknownExtra):
                env.log.error('Skipping "%s": (unknown extra "%s")', item, ue)
            else:
                env.log.error('Skipping "%s": %s', item,
                              exception_to_unicode(e, traceback=True))

        for dist, e in errors.iteritems():
            _log_error(dist, e)

        for entry in sorted(working_set.iter_entry_points(entry_point_name),
                            key=lambda entry: entry.name):
            env.log.debug('Loading %s from %s', entry.name, entry.dist.location)
            try:
                entry.load(require=True)
            except Exception, e:
                _log_error(entry, e)
            else:
                if os.path.dirname(entry.dist.location) == auto_enable:
                    _enable_plugin(env, entry.module_name)
    return _load_eggs

def load_py_files():
    """Loader that look for Python source files in the plugins directories,
    which simply get imported, thereby registering them with the component
    manager if they define any components.
    """
    def _load_py_files(env, search_path, auto_enable=None):
        for path in search_path:
            plugin_files = glob(os.path.join(path, '*.py'))
            for plugin_file in plugin_files:
                try:
                    plugin_name = os.path.basename(plugin_file[:-3])
                    env.log.debug('Loading file plugin %s from %s' % \
                                  (plugin_name, plugin_file))
                    if plugin_name not in sys.modules:
                        module = imp.load_source(plugin_name, plugin_file)
                    if path == auto_enable:
                        _enable_plugin(env, plugin_name)
                except Exception, e:
                    env.log.error('Failed to load plugin from %s: %s',
                                  plugin_file,
                                  exception_to_unicode(e, traceback=True))

    return _load_py_files

def get_plugins_dir(env):
    """Return the path to the `plugins` directory of the environment."""
    plugins_dir = os.path.realpath(os.path.join(env.path, 'plugins'))
    return os.path.normcase(plugins_dir)

def load_components(env, extra_path=None, loaders=(load_eggs('trac.plugins'),
                                                   load_py_files())):
    """Load all plugin components found on the given search path."""
    plugins_dir = get_plugins_dir(env)
    search_path = [plugins_dir]
    if extra_path:
        search_path += list(extra_path)

    for loadfunc in loaders:
        loadfunc(env, search_path, auto_enable=plugins_dir)

def get_plugin_info(env, include_core=False):
    """Return package information about Trac core and installed plugins."""
    path_sources = {}

    def find_distribution(module):
        name = module.__name__
        path = get_module_path(module)
        sources = path_sources.get(path)
        if sources is None:
            sources = path_sources[path] = get_sources(path)
        dist = sources.get(name.replace('.', '/') + '.py')
        if dist is None:
            dist = sources.get(name.replace('.', '/') + '/__init__.py')
        if dist is None:
            # This is a plain Python source file, not an egg
            dist = pkg_resources.Distribution(project_name=name,
                                              version='',
                                              location=module.__file__)
        return dist

    plugins_dir = get_plugins_dir(env)
    plugins = {}
    from trac.core import ComponentMeta
    for component in ComponentMeta._components:
        module = sys.modules[component.__module__]

        dist = find_distribution(module)
        plugin_filename = None
        if os.path.realpath(os.path.dirname(dist.location)) == plugins_dir:
            plugin_filename = os.path.basename(dist.location)

        if dist.project_name not in plugins:
            readonly = True
            if plugin_filename and os.access(dist.location,
                                             os.F_OK + os.W_OK):
                readonly = False
            # retrieve plugin metadata
            info = get_pkginfo(dist)
            if not info:
                info = {}
                for k in ('author', 'author_email', 'home_page', 'url',
                          'license', 'trac'):
                    v = getattr(module, k, '')
                    if v and isinstance(v, basestring):
                        if k == 'home_page' or k == 'url':
                            k = 'home_page'
                            v = v.replace('$', '').replace('URL: ', '')
                        else:
                            v = to_unicode(v)
                        info[k] = v
            else:
                # Info found; set all those fields to "None" that have the
                # value "UNKNOWN" as this is the value for fields that
                # aren't specified in "setup.py"
                for k in info:
                    if info[k] == 'UNKNOWN':
                        info[k] = ''
                    else:
                        # Must be encoded as unicode as otherwise Genshi
                        # may raise a "UnicodeDecodeError".
                        info[k] = to_unicode(info[k])

            # retrieve plugin version info
            version = dist.version
            if not version:
                version = (getattr(module, 'version', '') or
                           getattr(module, 'revision', ''))
                # special handling for "$Rev$" strings
                version = version.replace('$', '').replace('Rev: ', 'r')
            plugins[dist.project_name] = {
                'name': dist.project_name, 'version': version,
                'path': dist.location, 'plugin_filename': plugin_filename,
                'readonly': readonly, 'info': info, 'modules': {},
            }
        modules = plugins[dist.project_name]['modules']
        if module.__name__ not in modules:
            summary, description = get_doc(module)
            plugins[dist.project_name]['modules'][module.__name__] = {
                'summary': summary, 'description': description,
                'components': {},
            }
        full_name = module.__name__ + '.' + component.__name__
        summary, description = get_doc(component)
        c = component
        if c in env and not issubclass(c, env.__class__):
            c = component(env)
        modules[module.__name__]['components'][component.__name__] = {
            'full_name': full_name,
            'summary': summary, 'description': description,
            'enabled': env.is_component_enabled(component),
            'required': getattr(c, 'required', False),
        }
    if not include_core:
        for name in plugins.keys():
            if name.lower() == 'trac':
                plugins.pop(name)
    return sorted(plugins.itervalues(),
                  key=lambda p: (p['name'].lower() != 'trac',
                                 p['name'].lower()))

def match_plugins_to_frames(plugins, frames):
    """Add a `frame_idx` element to plugin information as returned by
    `get_plugin_info()`, containing the index of the highest frame in the
    list that was located in the plugin.
    """
    egg_frames = [(i, f) for i, f in enumerate(frames)
                  if f['filename'].startswith('build/')]

    def find_egg_frame_index(plugin):
        for dist in pkg_resources.find_distributions(plugin['path'],
                                                     only=True):
            try:
                sources = dist.get_metadata('SOURCES.txt')
                for src in sources.splitlines():
                    if src.endswith('.py'):
                        nsrc = src.replace('\\', '/')
                        for i, f in egg_frames:
                            if f['filename'].endswith(nsrc):
                                plugin['frame_idx'] = i
                                return
            except KeyError:
                pass    # Metadata not found

    for plugin in plugins:
        base, ext = os.path.splitext(plugin['path'].replace('\\', '/'))
        if ext == '.egg' and egg_frames:
            find_egg_frame_index(plugin)
        else:
            for i, f in enumerate(frames):
                if f['filename'].startswith(base):
                    plugin['frame_idx'] = i
                    break
<|endoftext|>


<commit_before>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016-2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

"""
Feature extraction

Usage:
  pyannote-speech-feature [--robust --database=<db.yml>] <experiment_dir> <database.task.protocol>
  pyannote-speech-feature check [--database=<db.yml>] <experiment_dir> <database.task.protocol>
  pyannote-speech-feature -h | --help
  pyannote-speech-feature --version

Options:
  <experiment_dir>           Set experiment root directory. This script expects
                             a configuration file called "config.yml" to live
                             in this directory. See "Configuration file"
                             section below for more details.
  <database.task.protocol>   Set evaluation protocol (e.g. "Etape.SpeakerDiarization.TV")
  --database=<db.yml>        Path to database configuration file.
                             [default: ~/.pyannote/db.yml]
  --robust                   When provided, skip files for which feature extraction fails.
  -h --help                  Show this screen.
  --version                  Show version.

Database configuration file:
    The database configuration provides details as to where actual files are
    stored. See `pyannote.database.util.FileFinder` docstring for more
    information on the expected format.

Configuration file:
    The configuration of each experiment is described in a file called
    <experiment_dir>/config.yml, that describes the feature extraction process
    (e.g. MFCCs).

    ................... <experiment_dir>/config.yml ...................
    feature_extraction:
       name: YaafeMFCC
       params:
          e: False                   # this experiments relies
          De: True                   # on 11 MFCC coefficients
          DDe: True                  # with 1st and 2nd derivatives
          D: True                    # without energy, but with
          DD: True                   # energy derivatives
    ...................................................................

"""

import yaml
import h5py
import os.path
import numpy as np
from docopt import docopt

import pyannote.core
import pyannote.database
from pyannote.database import FileFinder
from pyannote.database import get_unique_identifier
from pyannote.database import get_protocol

from pyannote.audio.util import mkdir_p
from pyannote.audio.features.utils import Precomputed
from pyannote.audio.features.utils import get_audio_duration
from pyannote.audio.features.utils import PyannoteFeatureExtractionError

def extract(protocol_name, file_finder, experiment_dir, robust=False):

    protocol = get_protocol(protocol_name, progress=False)

    # load configuration file
    config_yml = experiment_dir + '/config.yml'
    with open(config_yml, 'r') as fp:
        config = yaml.load(fp)

    feature_extraction_name = config['feature_extraction']['name']
    features = __import__('pyannote.audio.features',
                          fromlist=[feature_extraction_name])
    FeatureExtraction = getattr(features, feature_extraction_name)
    feature_extraction = FeatureExtraction(
        **config['feature_extraction'].get('params', {}))

    sliding_window = feature_extraction.sliding_window()
    dimension = feature_extraction.dimension()

    # create metadata file at root that contains
    # sliding window and dimension information
    path = Precomputed.get_config_path(experiment_dir)
    f = h5py.File(path)
    f.attrs['start'] = sliding_window.start
    f.attrs['duration'] = sliding_window.duration
    f.attrs['step'] = sliding_window.step
    f.attrs['dimension'] = dimension
    f.close()

    for subset in ['development', 'test', 'train']:

        try:
            protocol.progress = False
            file_generator = getattr(protocol, subset)()
            first_item = next(file_generator)
        except NotImplementedError as e:
            continue

        protocol.progress = True
        file_generator = getattr(protocol, subset)()

        for current_file in file_generator:

            try:
                current_file['audio'] = file_finder(current_file)
            except ValueError as e:
                if not robust:
                    raise e
                print(e)
                continue

            uri = get_unique_identifier(current_file)
            path = Precomputed.get_path(experiment_dir, current_file)

            if os.path.exists(path):
                continue

            try:
                features = feature_extraction(current_file)
            except PyannoteFeatureExtractionError as e:
                msg = 'Feature extraction failed for file "{uri}".'
                msg = msg.format(uri=uri)
                if not robust:
                    raise PyannoteFeatureExtractionError(msg)
                print(msg)
                continue

            if features is None:
                msg = 'Feature extraction returned None for file "{uri}".'
                msg = msg.format(uri=uri)
                if not robust:
                    raise PyannoteFeatureExtractionError(msg)
                print(msg)
                continue

            data = features.data

            if np.any(np.isnan(data)):
                msg = 'Feature extraction returned NaNs for file "{uri}".'
                msg = msg.format(uri=uri)
                if not robust:
                    raise PyannoteFeatureExtractionError(msg)
                print(msg)
                continue

            # create parent directory
            mkdir_p(os.path.dirname(path))

            f = h5py.File(path)
            f.attrs['start'] = sliding_window.start
            f.attrs['duration'] = sliding_window.duration
            f.attrs['step'] = sliding_window.step
            f.attrs['dimension'] = dimension
            f.create_dataset('features', data=data)
            f.close()

def check(protocol_name, file_finder, experiment_dir):

    protocol = get_protocol(protocol_name)
    precomputed = Precomputed(experiment_dir)

    for subset in ['development', 'test', 'train']:

        try:
            file_generator = getattr(protocol, subset)()
            first_item = next(file_generator)
        except NotImplementedError as e:
            continue

        for current_file in getattr(protocol, subset)():

            try:
                audio = file_finder(current_file)
                current_file['audio'] = audio
            except ValueError as e:
                print(e)
                continue

            duration = get_audio_duration(current_file)

            try:
                features = precomputed(current_file)
            except PyannoteFeatureExtractionError as e:
                print(e)
                continue

            if not np.isclose(duration,
                              features.getExtent().duration,
                              atol=1.):
                uri = get_unique_identifier(current_file)
                print('Duration mismatch for "{uri}"'.format(uri=uri))

def main():

    arguments = docopt(__doc__, version='Feature extraction')

    db_yml = os.path.expanduser(arguments['--database'])
    file_finder = FileFinder(db_yml)

    protocol_name = arguments['<database.task.protocol>']
    experiment_dir = arguments['<experiment_dir>']

    if arguments['check']:
        check(protocol_name, file_finder, experiment_dir)
    else:
        robust = arguments['--robust']
        extract(protocol_name, file_finder, experiment_dir, robust=robust)
<commit_msg>feat: add parallel processing to pyannote-speech-features<commit_after>#!/usr/bin/env python
# encoding: utf-8

# The MIT License (MIT)

# Copyright (c) 2016-2017 CNRS

# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:

# The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.

# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.

# AUTHORS
# Hervé BREDIN - http://herve.niderb.fr

"""
Feature extraction

Usage:
  pyannote-speech-feature [--robust --parallel --database=<db.yml>] <experiment_dir> <database.task.protocol>
  pyannote-speech-feature check [--database=<db.yml>] <experiment_dir> <database.task.protocol>
  pyannote-speech-feature -h | --help
  pyannote-speech-feature --version

Options:
  <experiment_dir>           Set experiment root directory. This script expects
                             a configuration file called "config.yml" to live
                             in this directory. See "Configuration file"
                             section below for more details.
  <database.task.protocol>   Set evaluation protocol (e.g. "Etape.SpeakerDiarization.TV")
  --database=<db.yml>        Path to database configuration file.
                             [default: ~/.pyannote/db.yml]
  --robust                   When provided, skip files for which feature extraction fails.
  --parallel                 When provided, process files in parallel.
  -h --help                  Show this screen.
  --version                  Show version.

Database configuration file:
    The database configuration provides details as to where actual files are
    stored. See `pyannote.database.util.FileFinder` docstring for more
    information on the expected format.

Configuration file:
    The configuration of each experiment is described in a file called
    <experiment_dir>/config.yml, that describes the feature extraction process
    (e.g. MFCCs).

    ................... <experiment_dir>/config.yml ...................
    feature_extraction:
       name: YaafeMFCC
       params:
          e: False                   # this experiments relies
          De: True                   # on 11 MFCC coefficients
          DDe: True                  # with 1st and 2nd derivatives
          D: True                    # without energy, but with
          DD: True                   # energy derivatives
    ...................................................................

"""

import yaml
import h5py
import os.path
import numpy as np
import functools
import itertools
from docopt import docopt

import pyannote.core
import pyannote.database
from pyannote.database import FileFinder
from pyannote.database import get_unique_identifier
from pyannote.database import get_protocol

from pyannote.audio.util import mkdir_p
from pyannote.audio.features.utils import Precomputed
from pyannote.audio.features.utils import get_audio_duration
from pyannote.audio.features.utils import PyannoteFeatureExtractionError

from multiprocessing import cpu_count, Pool

def init_feature_extraction(experiment_dir):

    # load configuration file
    config_yml = experiment_dir + '/config.yml'
    with open(config_yml, 'r') as fp:
        config = yaml.load(fp)

    feature_extraction_name = config['feature_extraction']['name']
    features = __import__('pyannote.audio.features',
                          fromlist=[feature_extraction_name])
    FeatureExtraction = getattr(features, feature_extraction_name)
    feature_extraction = FeatureExtraction(
        **config['feature_extraction'].get('params', {}))

    return feature_extraction

def process_current_file(current_file, file_finder=None, experiment_dir=None,
                         feature_extraction=None):

    try:
        current_file['audio'] = file_finder(current_file)
    except ValueError as e:
        if not robust:
            raise PyannoteFeatureExtractionError(*e.args)
        return e

    uri = get_unique_identifier(current_file)
    path = Precomputed.get_path(experiment_dir, current_file)

    if os.path.exists(path):
        return

    try:
        features = feature_extraction(current_file)
    except PyannoteFeatureExtractionError as e:
        msg = 'Feature extraction failed for file "{uri}".'
        return msg.format(uri=uri)

    if features is None:
        msg = 'Feature extraction returned None for file "{uri}".'
        return msg.format(uri=uri)

    data = features.data

    if np.any(np.isnan(data)):
        msg = 'Feature extraction returned NaNs for file "{uri}".'
        return msg.format(uri=uri)

    # create parent directory
    mkdir_p(os.path.dirname(path))

    sliding_window = feature_extraction.sliding_window()
    dimension = feature_extraction.dimension()

    f = h5py.File(path)

    f.attrs['start'] = sliding_window.start
    f.attrs['duration'] = sliding_window.duration
    f.attrs['step'] = sliding_window.step
    f.attrs['dimension'] = dimension
    f.create_dataset('features', data=data)
    f.close()

    return

def helper_extract(current_file, file_finder=None, experiment_dir=None,
                   config_yml=None, feature_extraction=None, robust=False):

    if feature_extraction is None:
        feature_extraction = init_feature_extraction(config_yml, experiment_dir)

    return process_current_file(current_file, file_finder=file_finder,
                                experiment_dir=experiment_dir,
                                feature_extraction=feature_extraction)

def extract(protocol_name, file_finder, experiment_dir,
            robust=False, parallel=False):

    protocol = get_protocol(protocol_name, progress=False)

    # load configuration file
    config_yml = experiment_dir + '/config.yml'
    with open(config_yml, 'r') as fp:
        config = yaml.load(fp)

    feature_extraction_name = config['feature_extraction']['name']
    features = __import__('pyannote.audio.features',
                          fromlist=[feature_extraction_name])
    FeatureExtraction = getattr(features, feature_extraction_name)
    feature_extraction = FeatureExtraction(
        **config['feature_extraction'].get('params', {}))

    sliding_window = feature_extraction.sliding_window()
    dimension = feature_extraction.dimension()

    # create metadata file at root that contains
    # sliding window and dimension information
    path = Precomputed.get_config_path(experiment_dir)
    f = h5py.File(path)
    f.attrs['start'] = sliding_window.start
    f.attrs['duration'] = sliding_window.duration
    f.attrs['step'] = sliding_window.step
    f.attrs['dimension'] = dimension
    f.close()

    if parallel:

        extract_one = functools.partial(helper_extract,
                                        file_finder=file_finder,
                                        experiment_dir=experiment_dir,
                                        config_yml=config_yml,
                                        robust=robust)

        n_jobs = cpu_count()
        pool = Pool(n_jobs)
        imap = pool.imap

    else:

        feature_extraction = init_feature_extraction(experiment_dir)
        extract_one = functools.partial(helper_extract,
                                        file_finder=file_finder,
                                        experiment_dir=experiment_dir,
                                        feature_extraction=feature_extraction,
                                        robust=robust)

        imap = itertools.imap

    for subset in ['development', 'test', 'train']:

        try:
            protocol.progress = False
            file_generator = getattr(protocol, subset)()
            first_item = next(file_generator)
        except NotImplementedError as e:
            continue

        protocol.progress = True
        file_generator = getattr(protocol, subset)()

        for result in imap(extract_one, file_generator):
            if result is None:
                continue
            print(result)

def check(protocol_name, file_finder, experiment_dir):

    protocol = get_protocol(protocol_name)
    precomputed = Precomputed(experiment_dir)

    for subset in ['development', 'test', 'train']:

        try:
            file_generator = getattr(protocol, subset)()
            first_item = next(file_generator)
        except NotImplementedError as e:
            continue

        for current_file in getattr(protocol, subset)():

            try:
                audio = file_finder(current_file)
                current_file['audio'] = audio
            except ValueError as e:
                print(e)
                continue

            duration = get_audio_duration(current_file)

            try:
                features = precomputed(current_file)
            except PyannoteFeatureExtractionError as e:
                print(e)
                continue

            if not np.isclose(duration,
                              features.getExtent().duration,
                              atol=1.):
                uri = get_unique_identifier(current_file)
                print('Duration mismatch for "{uri}"'.format(uri=uri))

def main():

    arguments = docopt(__doc__, version='Feature extraction')

    db_yml = os.path.expanduser(arguments['--database'])
    file_finder = FileFinder(db_yml)

    protocol_name = arguments['<database.task.protocol>']
    experiment_dir = arguments['<experiment_dir>']

    if arguments['check']:
        check(protocol_name, file_finder, experiment_dir)
    else:
        robust = arguments['--robust']
        parallel = arguments['--parallel']
        extract(protocol_name, file_finder, experiment_dir,
                robust=robust, parallel=parallel)
<|endoftext|>


<commit_before>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.55.0-dev.52'
'''The full version of MPF.'''

__short_version__ = '0.55'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

# pylint: disable-msg=invalid-name
version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

# pylint: disable-msg=invalid-name
extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

if "dev" in __version__:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/dev/logs/{}.html"
else:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/{}/logs/{{}}.html".format(__short_version__)

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version',
           'log_url']
<commit_msg>0.55.0-dev.53<commit_after>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.55.0-dev.53'
'''The full version of MPF.'''

__short_version__ = '0.55'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

# pylint: disable-msg=invalid-name
version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

# pylint: disable-msg=invalid-name
extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

if "dev" in __version__:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/dev/logs/{}.html"
else:
    # pylint: disable-msg=invalid-name
    log_url = "https://docs.missionpinball.org/en/{}/logs/{{}}.html".format(__short_version__)

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version',
           'log_url']
<|endoftext|>


<commit_before>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.50.0-dev.69'
'''The full version of MPF.'''

__short_version__ = '0.50'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version']
<commit_msg>dev-70<commit_after>"""Holds various Version strings of MPF.

This modules holds the MPF version strings, including the version of BCP it
needs and the config file version it needs.

It's used internally for all sorts of things, from printing the output of the
`mpf --version` command, to making sure any processes connected via BCP are
the proper versions, to automatically triggering new builds and deployments to
PyPI.

"""

__version__ = '0.50.0-dev.70'
'''The full version of MPF.'''

__short_version__ = '0.50'
'''The major.minor version of MPF.'''

__bcp_version__ = '1.1'
'''The version of BCP this build of MPF uses.'''

__config_version__ = '5'
'''The config file version this build of MPF uses.'''

__show_version__ = '5'
'''The show format version this build of MPF uses.'''

version = "MPF v{}".format(__version__)
'''A friendly version string for this build of MPF.'''

extended_version = "MPF v{}, Config version:{}, Show version: {}, " \
                   "BCP version:{}".format(__version__, __config_version__,
                                           __show_version__, __bcp_version__)
'''An extended version string that includes the MPF version, show version,
and BCP versions used in this build of MPF.'''

__api__ = ['version',
           '__short_version__',
           '__bcp_version__',
           '__config_version__',
           '__show_version__',
           'version',
           'extended_version']
<|endoftext|>