<commit_before># Python Version: 3.x
import onlinejudge
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import sys
import time
import shutil
import subprocess
import os
import re
from typing import *
if TYPE_CHECKING:
    import argparse

default_url_opener = [ 'sensible-browser', 'xdg-open', 'open' ]

def submit(args: 'argparse.Namespace') -> None:
    # parse url
    problem = onlinejudge.dispatch.problem_from_url(args.url)
    if problem is None:
        sys.exit(1)

    # read code
    with open(args.file) as fh:
        code = fh.buffer.read()
    format_config = {
        'dos2unix': args.format_dos2unix or args.golf,
        'rstrip': args.format_dos2unix or args.golf,
    }
    code = format_code(code, **format_config)

    # report code
    try:
        s = code.decode()
    except UnicodeDecodeError as e:
        log.failure('%s: %s', e.__class__.__name__, str(e))
        s = repr(code)[ 1 : ]
    log.info('code (%d byte):', len(code))
    log.emit(log.bold(s))

    with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess:
        # guess or select language ids
        langs = problem.get_language_dict(session=sess)
        matched_lang_ids: Optional[List[str]] = None
        if args.guess:
            kwargs = {
                'language_dict': langs,
                'cxx_latest': args.guess_cxx_latest,
                'cxx_compiler': args.guess_cxx_compiler,
                'python_version': args.guess_python_version,
                'python_interpreter': args.guess_python_interpreter,
            }
            matched_lang_ids = guess_lang_ids_of_file(args.file, code, **kwargs)
            if not matched_lang_ids:
                matched_lang_ids = list(langs.keys())
            if args.language is not None:
                matched_lang_ids = select_ids_of_matched_languages(args.language.split(), matched_lang_ids, language_dict=langs)
        else:
            if args.language is None:
                matched_lang_ids = None
            elif args.language in langs:
                matched_lang_ids = [ args.language ]
            else:
                matched_lang_ids = select_ids_of_matched_languages(args.language.split(), list(langs.keys()), language_dict=langs)

        # report selected language ids
        if matched_lang_ids is not None and len(matched_lang_ids) == 1:
            args.language = matched_lang_ids[0]
            log.info('choosed language: %s (%s)', args.language, langs[args.language]['description'])
        else:
            if matched_lang_ids is None:
                log.error('language is unknown')
                log.info('supported languages are:')
            elif len(matched_lang_ids) == 0:
                log.error('no languages are matched')
                log.info('supported languages are:')
            else:
                log.error('Matched languages were not narrowed down to one.')
                log.info('You have to choose:')
            for lang_id in sorted(matched_lang_ids or langs.keys()):
                log.emit('%s (%s)', lang_id, langs[lang_id]['description'])
            sys.exit(1)

        # confirm
        if args.wait:
            log.status('sleep(%.2f)', args.wait)
            time.sleep(args.wait)
        if not args.yes:
            sys.stdout.write('Are you sure? [y/N] ')
            sys.stdout.flush()
            c = sys.stdin.read(1)
            if c != 'y':
                log.info('terminated.')
                return

        # submit
        kwargs = {}
        if problem.get_service().get_name() == 'topcoder':
            if args.full_submission:
                kwargs['kind'] = 'full'
            else:
                kwargs['kind'] = 'example'
        try:
            submission = problem.submit(code, language=args.language, session=sess, **kwargs)  # type: ignore
        except onlinejudge.problem.SubmissionError:
            log.failure('submission failed')
            return

        # show result
        if args.open:
            if args.open_browser:
                browser = args.open_browser
            else:
                for browser in default_url_opener:
                    if shutil.which(browser):
                        break
                else:
                    browser = None
                    log.failure('couldn\'t find browsers to open the url. please specify a browser')
            if browser:
                log.info('open the submission page with: %s', browser)
                subprocess.check_call([ browser, submission.get_url() ], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)

def select_ids_of_matched_languages(words: List[str], lang_ids: List[str], language_dict, split: bool = False, remove: bool = False) -> List[str]:
    result = []
    for lang_id in lang_ids:
        desc = language_dict[lang_id]['description'].lower()
        if split:
            desc = desc.split()
        pred = all([ word.lower() in desc for word in words ])
        if remove:
            pred = not pred
        if pred:
            result.append(lang_id)
    return result

def guess_lang_ids_of_file(filename: str, code: bytes, language_dict, cxx_latest: bool = False, cxx_compiler: str = 'all', python_version: str = 'all', python_interpreter: str = 'all') -> List[str]:
    assert cxx_compiler.lower() in ( 'gcc', 'clang', 'all' )
    assert python_version.lower() in ( '2', '3', 'auto', 'all' )
    assert python_interpreter.lower() in ( 'cpython', 'pypy', 'all' )

    select = lambda word, lang_ids, **kwargs: select_ids_of_matched_languages([ word ], lang_ids, language_dict=language_dict, **kwargs)
    _, ext = os.path.splitext(filename)
    lang_ids = language_dict.keys()

    log.debug('file extension: %s', ext)
    ext = ext.lstrip('.')

    if ext in ( 'cpp', 'cxx', 'cc', 'C' ):
        log.debug('language guessing: C++')
        # memo: https://stackoverflow.com/questions/1545080/c-code-file-extension-cc-vs-cpp
        lang_ids = select('c++', lang_ids)
        if not lang_ids:
            return []

        # compiler
        if select('gcc', lang_ids) and select('clang', lang_ids):
            log.info('both GCC and Clang are available for C++ compiler')
            if cxx_compiler.lower() == 'gcc':
                log.info('use: GCC')
                lang_ids = select('gcc', lang_ids)
            elif cxx_compiler.lower() == 'clang':
                log.info('use: Clang')
                lang_ids = select('clang', lang_ids)
            else:
                assert cxx_compiler.lower() == 'all'

        # version
        if cxx_latest:
            saved_ids = lang_ids
            lang_ids = []
            for compiler in ( None, 'gcc', 'clang' ):  # use the latest for each compiler
                version_of = {}
                ids = select(compiler, saved_ids) if compiler else saved_ids
                if not ids:
                    continue
                for lang_id in ids:
                    m = re.search(r'c\+\+\w\w', language_dict[lang_id]['description'].lower())
                    if m:
                        version_of[lang_id] = m[0]
                ids.sort(key=lambda lang_id: version_of.get(lang_id, ''))
                lang_ids += [ ids[-1] ]  # since C++11 < C++1y < ... as strings
            lang_ids = list(set(lang_ids))

        assert lang_ids
        return lang_ids

    elif ext == 'py':
        log.debug('language guessing: Python')
        if select('pypy', language_dict.keys()):
            log.info('PyPy is available for Python interpreter')

        # interpreter
        lang_ids = []
        if python_interpreter.lower() in ( 'cpython', 'all' ):
            lang_ids += select('python', language_dict.keys())
        elif python_interpreter.lower() in ( 'pypy', 'all' ) or not lang_ids:
            lang_ids += select('pypy', language_dict.keys())

        # version
        if select('python2', lang_ids) and select('python3', lang_ids):
            log.info('both Python2 and Python3 are available for version of Python')
            if python_version in ( '2', '3' ):
                versions = [ int(python_version) ]
            elif python_version == 'all':
                versions = [ 2, 3 ]
            else:
                assert python_version == 'auto'
                lines = code.splitlines()
                if code.startswith(b'#!'):
                    s = lines[0]  # use shebang
                else:
                    s = b'\n'.join(lines[: 5] + lines[-5 :])  # use modelines
                versions = []
                for version in ( 2, 3 ):
                    if re.search(r'python ?%d'.encode() % version, s.lower()):
                        versions += [ version ]
                if not versions:
                    versions = [ 2, 3 ]
            log.info('use: %s', ', '.join(map(str, versions)))

            saved_ids = lang_ids
            lang_ids = []
            for version in versions:
                lang_ids += select('python%d'  % version, saved_ids)
                lang_ids += select('python %d' % version, saved_ids)

        lang_ids = list(set(lang_ids))
        return lang_ids

    else:
        log.debug('language guessing: othres')
        table: List[Dict[str, Any]] = [
             { 'names': [ 'awk'                   ], 'exts': [ 'awk'       ] },
             { 'names': [ 'bash'                  ], 'exts': [ 'sh'        ] },
             { 'names': [ 'brainfuck'             ], 'exts': [ 'bf'        ] },
             { 'names': [ 'c#'                    ], 'exts': [ 'cs'        ] },
             { 'names': [ 'c'                     ], 'exts': [ 'c'         ], 'split': True },
             { 'names': [ 'd'                     ], 'exts': [ 'd'         ], 'split': True },
             { 'names': [ 'f#'                    ], 'exts': [ 'fs'        ] },
             { 'names': [ 'fortran'               ], 'exts': [ 'for', 'f', 'f90', 'f95', 'f03' ] },
             { 'names': [ 'go'                    ], 'exts': [ 'go'        ], 'split': True },
             { 'names': [ 'haskell'               ], 'exts': [ 'hs'        ] },
             { 'names': [ 'java'                  ], 'exts': [ 'java'      ] },
             { 'names': [ 'javascript'            ], 'exts': [ 'js'        ] },
             { 'names': [ 'lua'                   ], 'exts': [ 'lua'       ] },
             { 'names': [ 'objective-c'           ], 'exts': [ 'm'         ] },
             { 'names': [ 'ocaml'                 ], 'exts': [ 'ml'        ] },
             { 'names': [ 'octave'                ], 'exts': [ 'm'         ] },
             { 'names': [ 'pascal'                ], 'exts': [ 'pas'       ] },
             { 'names': [ 'perl6'                 ], 'exts': [ 'p6'        ] },
             { 'names': [ 'perl'                  ], 'exts': [ 'pl'        ], 'split': True },
             { 'names': [ 'php'                   ], 'exts': [ 'php'       ] },
             { 'names': [ 'ruby'                  ], 'exts': [ 'rb'        ] },
             { 'names': [ 'rust'                  ], 'exts': [ 'rs'        ] },
             { 'names': [ 'scala'                 ], 'exts': [ 'scala'     ] },
             { 'names': [ 'scheme'                ], 'exts': [ 'scm'       ] },
             { 'names': [ 'sed'                   ], 'exts': [ 'sed'       ] },
             { 'names': [ 'standard ml'           ], 'exts': [ 'sml'       ] },
             { 'names': [ 'swift'                 ], 'exts': [ 'swift'     ] },
             { 'names': [ 'text'                  ], 'exts': [ 'txt'       ] },
             { 'names': [ 'typescript'            ], 'exts': [ 'ts'        ] },
             { 'names': [ 'vim script'            ], 'exts': [ 'vim'       ] },
        ]

        lang_ids = []
        for data in table:
            if ext in data['exts']:
                for name in data['names']:
                        lang_ids += select(name, language_dict.keys(), split=data.get('split', False))
        return list(set(lang_ids))

def format_code(code: bytes, dos2unix: bool = False, rstrip: bool = False) -> bytes:
    if dos2unix:
        log.info('dos2unix...')
        code = code.replace(b'\r\n', b'\n')
    if rstrip:
        log.info('rstrip...')
        code = code.rstrip()
    return code
<commit_msg>#193: add file extensions for Perl and Perl 6<commit_after># Python Version: 3.x
import onlinejudge
import onlinejudge.implementation.utils as utils
import onlinejudge.implementation.logging as log
import sys
import time
import shutil
import subprocess
import os
import re
from typing import *
if TYPE_CHECKING:
    import argparse

default_url_opener = [ 'sensible-browser', 'xdg-open', 'open' ]

def submit(args: 'argparse.Namespace') -> None:
    # parse url
    problem = onlinejudge.dispatch.problem_from_url(args.url)
    if problem is None:
        sys.exit(1)

    # read code
    with open(args.file) as fh:
        code = fh.buffer.read()
    format_config = {
        'dos2unix': args.format_dos2unix or args.golf,
        'rstrip': args.format_dos2unix or args.golf,
    }
    code = format_code(code, **format_config)

    # report code
    try:
        s = code.decode()
    except UnicodeDecodeError as e:
        log.failure('%s: %s', e.__class__.__name__, str(e))
        s = repr(code)[ 1 : ]
    log.info('code (%d byte):', len(code))
    log.emit(log.bold(s))

    with utils.with_cookiejar(utils.new_default_session(), path=args.cookie) as sess:
        # guess or select language ids
        langs = problem.get_language_dict(session=sess)
        matched_lang_ids: Optional[List[str]] = None
        if args.guess:
            kwargs = {
                'language_dict': langs,
                'cxx_latest': args.guess_cxx_latest,
                'cxx_compiler': args.guess_cxx_compiler,
                'python_version': args.guess_python_version,
                'python_interpreter': args.guess_python_interpreter,
            }
            matched_lang_ids = guess_lang_ids_of_file(args.file, code, **kwargs)
            if not matched_lang_ids:
                matched_lang_ids = list(langs.keys())
            if args.language is not None:
                matched_lang_ids = select_ids_of_matched_languages(args.language.split(), matched_lang_ids, language_dict=langs)
        else:
            if args.language is None:
                matched_lang_ids = None
            elif args.language in langs:
                matched_lang_ids = [ args.language ]
            else:
                matched_lang_ids = select_ids_of_matched_languages(args.language.split(), list(langs.keys()), language_dict=langs)

        # report selected language ids
        if matched_lang_ids is not None and len(matched_lang_ids) == 1:
            args.language = matched_lang_ids[0]
            log.info('choosed language: %s (%s)', args.language, langs[args.language]['description'])
        else:
            if matched_lang_ids is None:
                log.error('language is unknown')
                log.info('supported languages are:')
            elif len(matched_lang_ids) == 0:
                log.error('no languages are matched')
                log.info('supported languages are:')
            else:
                log.error('Matched languages were not narrowed down to one.')
                log.info('You have to choose:')
            for lang_id in sorted(matched_lang_ids or langs.keys()):
                log.emit('%s (%s)', lang_id, langs[lang_id]['description'])
            sys.exit(1)

        # confirm
        if args.wait:
            log.status('sleep(%.2f)', args.wait)
            time.sleep(args.wait)
        if not args.yes:
            sys.stdout.write('Are you sure? [y/N] ')
            sys.stdout.flush()
            c = sys.stdin.read(1)
            if c != 'y':
                log.info('terminated.')
                return

        # submit
        kwargs = {}
        if problem.get_service().get_name() == 'topcoder':
            if args.full_submission:
                kwargs['kind'] = 'full'
            else:
                kwargs['kind'] = 'example'
        try:
            submission = problem.submit(code, language=args.language, session=sess, **kwargs)  # type: ignore
        except onlinejudge.problem.SubmissionError:
            log.failure('submission failed')
            return

        # show result
        if args.open:
            if args.open_browser:
                browser = args.open_browser
            else:
                for browser in default_url_opener:
                    if shutil.which(browser):
                        break
                else:
                    browser = None
                    log.failure('couldn\'t find browsers to open the url. please specify a browser')
            if browser:
                log.info('open the submission page with: %s', browser)
                subprocess.check_call([ browser, submission.get_url() ], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr)

def select_ids_of_matched_languages(words: List[str], lang_ids: List[str], language_dict, split: bool = False, remove: bool = False) -> List[str]:
    result = []
    for lang_id in lang_ids:
        desc = language_dict[lang_id]['description'].lower()
        if split:
            desc = desc.split()
        pred = all([ word.lower() in desc for word in words ])
        if remove:
            pred = not pred
        if pred:
            result.append(lang_id)
    return result

def guess_lang_ids_of_file(filename: str, code: bytes, language_dict, cxx_latest: bool = False, cxx_compiler: str = 'all', python_version: str = 'all', python_interpreter: str = 'all') -> List[str]:
    assert cxx_compiler.lower() in ( 'gcc', 'clang', 'all' )
    assert python_version.lower() in ( '2', '3', 'auto', 'all' )
    assert python_interpreter.lower() in ( 'cpython', 'pypy', 'all' )

    select = lambda word, lang_ids, **kwargs: select_ids_of_matched_languages([ word ], lang_ids, language_dict=language_dict, **kwargs)
    _, ext = os.path.splitext(filename)
    lang_ids = language_dict.keys()

    log.debug('file extension: %s', ext)
    ext = ext.lstrip('.')

    if ext in ( 'cpp', 'cxx', 'cc', 'C' ):
        log.debug('language guessing: C++')
        # memo: https://stackoverflow.com/questions/1545080/c-code-file-extension-cc-vs-cpp
        lang_ids = select('c++', lang_ids)
        if not lang_ids:
            return []

        # compiler
        if select('gcc', lang_ids) and select('clang', lang_ids):
            log.info('both GCC and Clang are available for C++ compiler')
            if cxx_compiler.lower() == 'gcc':
                log.info('use: GCC')
                lang_ids = select('gcc', lang_ids)
            elif cxx_compiler.lower() == 'clang':
                log.info('use: Clang')
                lang_ids = select('clang', lang_ids)
            else:
                assert cxx_compiler.lower() == 'all'

        # version
        if cxx_latest:
            saved_ids = lang_ids
            lang_ids = []
            for compiler in ( None, 'gcc', 'clang' ):  # use the latest for each compiler
                version_of = {}
                ids = select(compiler, saved_ids) if compiler else saved_ids
                if not ids:
                    continue
                for lang_id in ids:
                    m = re.search(r'c\+\+\w\w', language_dict[lang_id]['description'].lower())
                    if m:
                        version_of[lang_id] = m[0]
                ids.sort(key=lambda lang_id: version_of.get(lang_id, ''))
                lang_ids += [ ids[-1] ]  # since C++11 < C++1y < ... as strings
            lang_ids = list(set(lang_ids))

        assert lang_ids
        return lang_ids

    elif ext == 'py':
        log.debug('language guessing: Python')
        if select('pypy', language_dict.keys()):
            log.info('PyPy is available for Python interpreter')

        # interpreter
        lang_ids = []
        if python_interpreter.lower() in ( 'cpython', 'all' ):
            lang_ids += select('python', language_dict.keys())
        elif python_interpreter.lower() in ( 'pypy', 'all' ) or not lang_ids:
            lang_ids += select('pypy', language_dict.keys())

        # version
        if select('python2', lang_ids) and select('python3', lang_ids):
            log.info('both Python2 and Python3 are available for version of Python')
            if python_version in ( '2', '3' ):
                versions = [ int(python_version) ]
            elif python_version == 'all':
                versions = [ 2, 3 ]
            else:
                assert python_version == 'auto'
                lines = code.splitlines()
                if code.startswith(b'#!'):
                    s = lines[0]  # use shebang
                else:
                    s = b'\n'.join(lines[: 5] + lines[-5 :])  # use modelines
                versions = []
                for version in ( 2, 3 ):
                    if re.search(r'python ?%d'.encode() % version, s.lower()):
                        versions += [ version ]
                if not versions:
                    versions = [ 2, 3 ]
            log.info('use: %s', ', '.join(map(str, versions)))

            saved_ids = lang_ids
            lang_ids = []
            for version in versions:
                lang_ids += select('python%d'  % version, saved_ids)
                lang_ids += select('python %d' % version, saved_ids)

        lang_ids = list(set(lang_ids))
        return lang_ids

    else:
        log.debug('language guessing: othres')
        table: List[Dict[str, Any]] = [
             { 'names': [ 'awk'                   ], 'exts': [ 'awk'       ] },
             { 'names': [ 'bash'                  ], 'exts': [ 'sh'        ] },
             { 'names': [ 'brainfuck'             ], 'exts': [ 'bf'        ] },
             { 'names': [ 'c#'                    ], 'exts': [ 'cs'        ] },
             { 'names': [ 'c'                     ], 'exts': [ 'c'         ], 'split': True },
             { 'names': [ 'd'                     ], 'exts': [ 'd'         ], 'split': True },
             { 'names': [ 'f#'                    ], 'exts': [ 'fs'        ] },
             { 'names': [ 'fortran'               ], 'exts': [ 'for', 'f', 'f90', 'f95', 'f03' ] },
             { 'names': [ 'go'                    ], 'exts': [ 'go'        ], 'split': True },
             { 'names': [ 'haskell'               ], 'exts': [ 'hs'        ] },
             { 'names': [ 'java'                  ], 'exts': [ 'java'      ] },
             { 'names': [ 'javascript'            ], 'exts': [ 'js'        ] },
             { 'names': [ 'lua'                   ], 'exts': [ 'lua'       ] },
             { 'names': [ 'objective-c'           ], 'exts': [ 'm'         ] },
             { 'names': [ 'ocaml'                 ], 'exts': [ 'ml'        ] },
             { 'names': [ 'octave'                ], 'exts': [ 'm'         ] },
             { 'names': [ 'pascal'                ], 'exts': [ 'pas'       ] },
             { 'names': [ 'perl6'                 ], 'exts': [ 'p6', 'pl6', 'pm6' ] },
             { 'names': [ 'perl'                  ], 'exts': [ 'pl', 'pm'  ], 'split': True },
             { 'names': [ 'php'                   ], 'exts': [ 'php'       ] },
             { 'names': [ 'ruby'                  ], 'exts': [ 'rb'        ] },
             { 'names': [ 'rust'                  ], 'exts': [ 'rs'        ] },
             { 'names': [ 'scala'                 ], 'exts': [ 'scala'     ] },
             { 'names': [ 'scheme'                ], 'exts': [ 'scm'       ] },
             { 'names': [ 'sed'                   ], 'exts': [ 'sed'       ] },
             { 'names': [ 'standard ml'           ], 'exts': [ 'sml'       ] },
             { 'names': [ 'swift'                 ], 'exts': [ 'swift'     ] },
             { 'names': [ 'text'                  ], 'exts': [ 'txt'       ] },
             { 'names': [ 'typescript'            ], 'exts': [ 'ts'        ] },
             { 'names': [ 'vim script'            ], 'exts': [ 'vim'       ] },
        ]

        lang_ids = []
        for data in table:
            if ext in data['exts']:
                for name in data['names']:
                        lang_ids += select(name, language_dict.keys(), split=data.get('split', False))
        return list(set(lang_ids))

def format_code(code: bytes, dos2unix: bool = False, rstrip: bool = False) -> bytes:
    if dos2unix:
        log.info('dos2unix...')
        code = code.replace(b'\r\n', b'\n')
    if rstrip:
        log.info('rstrip...')
        code = code.rstrip()
    return code
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import sys

from weboob.capabilities.radio import ICapRadio
from weboob.capabilities.base import NotLoaded
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.media_player import InvalidMediaPlayer, MediaPlayer, MediaPlayerNotFound
from weboob.tools.application.formatters.iformatter import IFormatter

__all__ = ['Radioob']

class RadioListFormatter(IFormatter):
    MANDATORY_FIELDS = ('id', 'title', 'description')

    count = 0

    def flush(self):
        self.count = 0
        pass

    def format_dict(self, item):
        self.count += 1
        if self.interactive:
            backend = item['id'].split('@', 1)[1]
            result = u'%s* (%d) %s (%s)%s\n' % (ReplApplication.BOLD, self.count, item['title'], backend, ReplApplication.NC)
        else:
            result = u'%s* (%s) %s%s\n' % (ReplApplication.BOLD, item['id'], item['title'], ReplApplication.NC)
        result += '   %-30s' % item['description']
        if item['current'] is not NotLoaded:
            if item['current'].artist:
                result += ' (Current: %s - %s)' % (item['current'].artist, item['current'].title)
            else:
                result += ' (Current: %s)' % item['current'].title
        return result

class Radioob(ReplApplication):
    APPNAME = 'radioob'
    VERSION = '0.a'
    COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
    DESCRIPTION = 'Console application allowing to search for web radio stations, listen to them and get information ' \
                  'like the current song.'
    CAPS = ICapRadio
    EXTRA_FORMATTERS = {'radio_list': RadioListFormatter}
    COMMANDS_FORMATTERS = {'ls':     'radio_list',
                           'search': 'radio_list',
                          }

    def __init__(self, *args, **kwargs):
        ReplApplication.__init__(self, *args, **kwargs)
        self.player = MediaPlayer(self.logger)

    def main(self, argv):
        self.load_config()
        return ReplApplication.main(self, argv)

    def complete_play(self, text, line, *ignored):
        args = line.split(' ')
        if len(args) == 2:
            return self._complete_object()

    def do_play(self, _id):
        """
        play ID

        Play a radio with a found player.
        """
        if not _id:
            print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('play', short=True)
            return 2

        radio = self.get_object(_id, 'get_radio', ['streams'])
        if not radio:
            print >>sys.stderr, 'Radio not found: ' % _id
            return 1
        try:
            player_name = self.config.get('media_player')
            if not player_name:
                self.logger.debug(u'You can set the media_player key to the player you prefer in the radioob '
                                  'configuration file.')
            self.player.play(radio.streams[0], player_name=player_name)
        except (InvalidMediaPlayer, MediaPlayerNotFound), e:
            print '%s\nRadio URL: %s' % (e, radio.streams[0].url)

    def complete_info(self, text, line, *ignored):
        args = line.split(' ')
        if len(args) == 2:
            return self._complete_object()

    def do_info(self, _id):
        """
        info ID

        Get information about a radio.
        """
        if not _id:
            print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('info', short=True)
            return 2

        radio = self.get_object(_id, 'get_radio')
        if not radio:
            print >>sys.stderr, 'Radio not found:', _id
            return 3
        self.format(radio)
        self.flush()

    def do_search(self, pattern=None):
        """
        search PATTERN

        List radios matching a PATTERN.

        If PATTERN is not given, this command will list all the radios.
        """
        self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'All radios')
        self.change_path('/search')
        for backend, radio in self.do('iter_radios_search', pattern=pattern):
            self.add_object(radio)
            self.format(radio)
        self.flush()

<commit_msg>fix formatting error<commit_after># -*- coding: utf-8 -*-

# Copyright(C) 2010-2011 Romain Bignon
#
# This file is part of weboob.
#
# weboob is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# weboob is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with weboob. If not, see <http://www.gnu.org/licenses/>.

import sys

from weboob.capabilities.radio import ICapRadio
from weboob.capabilities.base import NotLoaded
from weboob.tools.application.repl import ReplApplication
from weboob.tools.application.media_player import InvalidMediaPlayer, MediaPlayer, MediaPlayerNotFound
from weboob.tools.application.formatters.iformatter import IFormatter

__all__ = ['Radioob']

class RadioListFormatter(IFormatter):
    MANDATORY_FIELDS = ('id', 'title', 'description')

    count = 0

    def flush(self):
        self.count = 0
        pass

    def format_dict(self, item):
        self.count += 1
        if self.interactive:
            backend = item['id'].split('@', 1)[1]
            result = u'%s* (%d) %s (%s)%s\n' % (ReplApplication.BOLD, self.count, item['title'], backend, ReplApplication.NC)
        else:
            result = u'%s* (%s) %s%s\n' % (ReplApplication.BOLD, item['id'], item['title'], ReplApplication.NC)
        result += '   %-30s' % item['description']
        if item['current'] is not NotLoaded:
            if item['current'].artist:
                result += ' (Current: %s - %s)' % (item['current'].artist, item['current'].title)
            else:
                result += ' (Current: %s)' % item['current'].title
        return result

class Radioob(ReplApplication):
    APPNAME = 'radioob'
    VERSION = '0.a'
    COPYRIGHT = 'Copyright(C) 2010-2011 Romain Bignon'
    DESCRIPTION = 'Console application allowing to search for web radio stations, listen to them and get information ' \
                  'like the current song.'
    CAPS = ICapRadio
    EXTRA_FORMATTERS = {'radio_list': RadioListFormatter}
    COMMANDS_FORMATTERS = {'ls':     'radio_list',
                           'search': 'radio_list',
                          }

    def __init__(self, *args, **kwargs):
        ReplApplication.__init__(self, *args, **kwargs)
        self.player = MediaPlayer(self.logger)

    def main(self, argv):
        self.load_config()
        return ReplApplication.main(self, argv)

    def complete_play(self, text, line, *ignored):
        args = line.split(' ')
        if len(args) == 2:
            return self._complete_object()

    def do_play(self, _id):
        """
        play ID

        Play a radio with a found player.
        """
        if not _id:
            print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('play', short=True)
            return 2

        radio = self.get_object(_id, 'get_radio', ['streams'])
        if not radio:
            print >>sys.stderr, 'Radio not found:', _id
            return 1
        try:
            player_name = self.config.get('media_player')
            if not player_name:
                self.logger.debug(u'You can set the media_player key to the player you prefer in the radioob '
                                  'configuration file.')
            self.player.play(radio.streams[0], player_name=player_name)
        except (InvalidMediaPlayer, MediaPlayerNotFound), e:
            print '%s\nRadio URL: %s' % (e, radio.streams[0].url)

    def complete_info(self, text, line, *ignored):
        args = line.split(' ')
        if len(args) == 2:
            return self._complete_object()

    def do_info(self, _id):
        """
        info ID

        Get information about a radio.
        """
        if not _id:
            print >>sys.stderr, 'This command takes an argument: %s' % self.get_command_help('info', short=True)
            return 2

        radio = self.get_object(_id, 'get_radio')
        if not radio:
            print >>sys.stderr, 'Radio not found:', _id
            return 3
        self.format(radio)
        self.flush()

    def do_search(self, pattern=None):
        """
        search PATTERN

        List radios matching a PATTERN.

        If PATTERN is not given, this command will list all the radios.
        """
        self.set_formatter_header(u'Search pattern: %s' % pattern if pattern else u'All radios')
        self.change_path('/search')
        for backend, radio in self.do('iter_radios_search', pattern=pattern):
            self.add_object(radio)
            self.format(radio)
        self.flush()

<|endoftext|>


<commit_before>#!/usr/local/bin/python3

"""
    Copyright (c) 2016-2020 Ad Schellevis <ad@opnsense.org>
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    watch dhcp lease file and build include file for unbound
"""
import os
import sys
import subprocess
import time
import tempfile
import argparse
import syslog
sys.path.insert(0, "/usr/local/opnsense/site-python")
from daemonize import Daemonize
import watchers.dhcpd

def unbound_control(commands, output_stream=None):
    """ execute (chrooted) unbound-control command
        :param commands: command list (parameters)
        :param output_stream: (optional)output stream
        :return: None
    """
    if output_stream is None:
        output_stream = open(os.devnull, 'w')
    subprocess.check_call(['/usr/sbin/chroot', '-u', 'unbound', '-g', 'unbound', '/',
                           '/usr/local/sbin/unbound-control', '-c', '/var/unbound/unbound.conf'] + commands,
                          stdout=output_stream, stderr=subprocess.STDOUT)
    output_stream.seek(0)

class UnboundLocalData:
    def __init__(self):
        self._map_by_address = dict()
        self._map_by_fqdn = dict()
        self.load()

    def load(self):
        self._map_by_address = dict()
        self._map_by_fqdn = dict()
        with tempfile.NamedTemporaryFile() as output_stream:
            unbound_control(['list_local_data'], output_stream)
            for line in output_stream:
                parts = line.decode().split()
                if len(parts) > 4 and parts[3] == 'A' and parts[4] != '0.0.0.0':
                    self.add_address(parts[4], parts[0][:-1])

    def add_address(self, address, fqdn):
        if address not in self._map_by_address:
            self._map_by_address[address] = list()
        self._map_by_address[address].append(fqdn)
        self._map_by_fqdn[fqdn] = address

    def all_fqdns(self, address, fqdn):
        result = set()
        if address in self._map_by_address:
            for unbfqdn in self._map_by_address[address]:
                result.add(unbfqdn)
        if fqdn in self._map_by_fqdn:
            result.add(fqdn)
        return result

    def cleanup(self, address, fqdn):
        if address in self._map_by_address:
            for rfqdn in self._map_by_address[address]:
                if rfqdn in self._map_by_fqdn:
                    del self._map_by_fqdn[rfqdn]
            del self._map_by_address[address]
        if fqdn in self._map_by_fqdn:
            if self._map_by_fqdn[fqdn] in self._map_by_address:
                del self._map_by_address[self._map_by_fqdn[fqdn]]
            del self._map_by_fqdn[fqdn]

    def is_equal(self, address, fqdn):
        tmp = self.all_fqdns(address, fqdn)
        return len(tmp) == 1 and fqdn in self._map_by_fqdn and self._map_by_fqdn[fqdn] == address

def run_watcher(target_filename, domain):
    # cleanup interval (seconds)
    cleanup_interval = 60

    # initiate lease watcher and setup cache
    dhcpdleases = watchers.dhcpd.DHCPDLease()
    cached_leases = dict()
    unbound_local_data = UnboundLocalData()

    # start watching dhcp leases
    last_cleanup = time.time()
    while True:
        dhcpd_changed = False
        for lease in dhcpdleases.watch():
            if 'ends' in lease and lease['ends'] > time.time() \
                    and 'client-hostname' in lease and 'address' in lease and lease['client-hostname']:
                cached_leases[lease['address']] = lease
                dhcpd_changed = True
        if time.time() - last_cleanup > cleanup_interval:
            # cleanup every x seconds
            last_cleanup = time.time()
            addresses = list(cached_leases)
            for address in addresses:
                if cached_leases[address]['ends'] < time.time():
                    syslog.syslog(
                        syslog.LOG_NOTICE,
                        "dhcpd expired %s @ %s" % (cached_leases[address]['client-hostname'], address)
                    )
                    unbound_control(['local_data_remove',  cached_leases[address]['client-hostname']])
                    del cached_leases[address]
                    fqdn = '%s.%s' % (cached_leases[address]['client-hostname'], domain)
                    if unbound_local_data.is_equal(address, fqdn):
                        unbound_local_data.cleanup(address, fqdn)
                    dhcpd_changed = True

        if dhcpd_changed:
            # dump dns output to target (used on initial startup, unbound_control is used as live feed)
            with open(target_filename, 'w') as unbound_conf:
                for address in cached_leases:
                    unbound_conf.write('local-data-ptr: "%s %s.%s"\n' % (
                        address, cached_leases[address]['client-hostname'], domain)
                    )
                    unbound_conf.write('local-data: "%s.%s IN A %s"\n' % (
                        cached_leases[address]['client-hostname'], domain, address)
                    )
            # signal unbound
            for address in cached_leases:
                fqdn = '%s.%s' % (cached_leases[address]['client-hostname'], domain)
                if not unbound_local_data.is_equal(address, fqdn):
                    for tmp_fqdn in unbound_local_data.all_fqdns(address, fqdn):
                        syslog.syslog(syslog.LOG_NOTICE, 'dhcpd entry changed %s @ %s.' % (tmp_fqdn, address))
                        unbound_control(['local_data_remove', tmp_fqdn])
                    unbound_local_data.cleanup(address, fqdn)
                    unbound_control(['local_data', address, 'PTR', fqdn])
                    unbound_control(['local_data', fqdn, 'IN A', address])
                    unbound_local_data.add_address(address, fqdn)

        # wait for next cycle
        time.sleep(1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--pid', help='pid file location', default='/var/run/unbound_dhcpd.pid')
    parser.add_argument('--target', help='target config file, used when unbound restarts',
                                    default='/var/unbound/dhcpleases.conf')
    parser.add_argument('--foreground', help='run in forground', default=False, action='store_true')
    parser.add_argument('--domain', help='domain to use',  default='local')
    inputargs = parser.parse_args()

    syslog.openlog('unbound', logoption=syslog.LOG_DAEMON, facility=syslog.LOG_LOCAL4)

    if inputargs.foreground:
        run_watcher(target_filename=inputargs.target, domain=inputargs.domain)
    else:
        syslog.syslog(syslog.LOG_NOTICE, 'daemonize unbound dhcpd watcher.')
        cmd  = lambda : run_watcher(target_filename=inputargs.target, domain=inputargs.domain)
        daemon = Daemonize(app="unbound_dhcpd", pid=inputargs.pid, action=cmd)
        daemon.start()
<commit_msg>Unbound: Fix bad dereference when DHCP lease expires (#4742)<commit_after>#!/usr/local/bin/python3

"""
    Copyright (c) 2016-2020 Ad Schellevis <ad@opnsense.org>
    All rights reserved.

    Redistribution and use in source and binary forms, with or without
    modification, are permitted provided that the following conditions are met:

    1. Redistributions of source code must retain the above copyright notice,
     this list of conditions and the following disclaimer.

    2. Redistributions in binary form must reproduce the above copyright
     notice, this list of conditions and the following disclaimer in the
     documentation and/or other materials provided with the distribution.

    THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES,
    INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
    AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
    AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
    OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
    SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
    INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
    CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
    ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
    POSSIBILITY OF SUCH DAMAGE.

    --------------------------------------------------------------------------------------
    watch dhcp lease file and build include file for unbound
"""
import os
import sys
import subprocess
import time
import tempfile
import argparse
import syslog
sys.path.insert(0, "/usr/local/opnsense/site-python")
from daemonize import Daemonize
import watchers.dhcpd

def unbound_control(commands, output_stream=None):
    """ execute (chrooted) unbound-control command
        :param commands: command list (parameters)
        :param output_stream: (optional)output stream
        :return: None
    """
    if output_stream is None:
        output_stream = open(os.devnull, 'w')
    subprocess.check_call(['/usr/sbin/chroot', '-u', 'unbound', '-g', 'unbound', '/',
                           '/usr/local/sbin/unbound-control', '-c', '/var/unbound/unbound.conf'] + commands,
                          stdout=output_stream, stderr=subprocess.STDOUT)
    output_stream.seek(0)

class UnboundLocalData:
    def __init__(self):
        self._map_by_address = dict()
        self._map_by_fqdn = dict()
        self.load()

    def load(self):
        self._map_by_address = dict()
        self._map_by_fqdn = dict()
        with tempfile.NamedTemporaryFile() as output_stream:
            unbound_control(['list_local_data'], output_stream)
            for line in output_stream:
                parts = line.decode().split()
                if len(parts) > 4 and parts[3] == 'A' and parts[4] != '0.0.0.0':
                    self.add_address(parts[4], parts[0][:-1])

    def add_address(self, address, fqdn):
        if address not in self._map_by_address:
            self._map_by_address[address] = list()
        self._map_by_address[address].append(fqdn)
        self._map_by_fqdn[fqdn] = address

    def all_fqdns(self, address, fqdn):
        result = set()
        if address in self._map_by_address:
            for unbfqdn in self._map_by_address[address]:
                result.add(unbfqdn)
        if fqdn in self._map_by_fqdn:
            result.add(fqdn)
        return result

    def cleanup(self, address, fqdn):
        if address in self._map_by_address:
            for rfqdn in self._map_by_address[address]:
                if rfqdn in self._map_by_fqdn:
                    del self._map_by_fqdn[rfqdn]
            del self._map_by_address[address]
        if fqdn in self._map_by_fqdn:
            if self._map_by_fqdn[fqdn] in self._map_by_address:
                del self._map_by_address[self._map_by_fqdn[fqdn]]
            del self._map_by_fqdn[fqdn]

    def is_equal(self, address, fqdn):
        tmp = self.all_fqdns(address, fqdn)
        return len(tmp) == 1 and fqdn in self._map_by_fqdn and self._map_by_fqdn[fqdn] == address

def run_watcher(target_filename, domain):
    # cleanup interval (seconds)
    cleanup_interval = 60

    # initiate lease watcher and setup cache
    dhcpdleases = watchers.dhcpd.DHCPDLease()
    cached_leases = dict()
    unbound_local_data = UnboundLocalData()

    # start watching dhcp leases
    last_cleanup = time.time()
    while True:
        dhcpd_changed = False
        for lease in dhcpdleases.watch():
            if 'ends' in lease and lease['ends'] > time.time() \
                    and 'client-hostname' in lease and 'address' in lease and lease['client-hostname']:
                cached_leases[lease['address']] = lease
                dhcpd_changed = True
        if time.time() - last_cleanup > cleanup_interval:
            # cleanup every x seconds
            last_cleanup = time.time()
            addresses = list(cached_leases)
            for address in addresses:
                if cached_leases[address]['ends'] < time.time():
                    syslog.syslog(
                        syslog.LOG_NOTICE,
                        "dhcpd expired %s @ %s" % (cached_leases[address]['client-hostname'], address)
                    )
                    unbound_control(['local_data_remove',  cached_leases[address]['client-hostname']])
                    fqdn = '%s.%s' % (cached_leases[address]['client-hostname'], domain)
                    if unbound_local_data.is_equal(address, fqdn):
                        unbound_local_data.cleanup(address, fqdn)
                    del cached_leases[address]
                    dhcpd_changed = True

        if dhcpd_changed:
            # dump dns output to target (used on initial startup, unbound_control is used as live feed)
            with open(target_filename, 'w') as unbound_conf:
                for address in cached_leases:
                    unbound_conf.write('local-data-ptr: "%s %s.%s"\n' % (
                        address, cached_leases[address]['client-hostname'], domain)
                    )
                    unbound_conf.write('local-data: "%s.%s IN A %s"\n' % (
                        cached_leases[address]['client-hostname'], domain, address)
                    )
            # signal unbound
            for address in cached_leases:
                fqdn = '%s.%s' % (cached_leases[address]['client-hostname'], domain)
                if not unbound_local_data.is_equal(address, fqdn):
                    for tmp_fqdn in unbound_local_data.all_fqdns(address, fqdn):
                        syslog.syslog(syslog.LOG_NOTICE, 'dhcpd entry changed %s @ %s.' % (tmp_fqdn, address))
                        unbound_control(['local_data_remove', tmp_fqdn])
                    unbound_local_data.cleanup(address, fqdn)
                    unbound_control(['local_data', address, 'PTR', fqdn])
                    unbound_control(['local_data', fqdn, 'IN A', address])
                    unbound_local_data.add_address(address, fqdn)

        # wait for next cycle
        time.sleep(1)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--pid', help='pid file location', default='/var/run/unbound_dhcpd.pid')
    parser.add_argument('--target', help='target config file, used when unbound restarts',
                                    default='/var/unbound/dhcpleases.conf')
    parser.add_argument('--foreground', help='run in forground', default=False, action='store_true')
    parser.add_argument('--domain', help='domain to use',  default='local')
    inputargs = parser.parse_args()

    syslog.openlog('unbound', logoption=syslog.LOG_DAEMON, facility=syslog.LOG_LOCAL4)

    if inputargs.foreground:
        run_watcher(target_filename=inputargs.target, domain=inputargs.domain)
    else:
        syslog.syslog(syslog.LOG_NOTICE, 'daemonize unbound dhcpd watcher.')
        cmd  = lambda : run_watcher(target_filename=inputargs.target, domain=inputargs.domain)
        daemon = Daemonize(app="unbound_dhcpd", pid=inputargs.pid, action=cmd)
        daemon.start()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
To use this script you'll need flac and lame
Usage :
flac2mp3 [origin directories containing flac files] mp3/repository/destination
"""
import getopt
from itertools import repeat
from logging import INFO
import logging
from multiprocessing import Pool
import multiprocessing
from posix import getcwd
from struct import unpack
import sys
from genericpath import isdir
import os
from os.path import dirname, join
from subprocess import Popen, PIPE

__author__ = 'bruno thomas'

logging.basicConfig(format='%(asctime)s [%(name)s] %(levelname)s: %(message)s')
LOGGER = logging.getLogger('flac2mp3')

LAME_COMMAND = 'lame --silent -V2 --vbr-new -q0 --lowpass 19.7 --resample 44100 --add-id3v2'
VOBIS_COMMENT = 4

class none_if_missing(dict):
    def __init__(self, seq=None, **kwargs): super(none_if_missing, self).__init__(seq, **kwargs)
    def __missing__(self, _):return None

vobis_comments_lame_opts_map = none_if_missing({
   'ARTIST'     : '--ta',
   'ALBUM'      : '--tl',
   'TITLE'      : '--tt',
   'DESCRIPTION': '--tc',
   'GENRE'      : '--tg',
   'DATE'       : '--ty',
   'TRACKNUMBER': '--tn',
   'TRACKTOTAL' : 'total'
})

def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    except NotImplementedError:
        return 1

def transcode(flac_file, mp3_file):
    tags = get_flac_tags(get_vobis_comment_bloc(flac_file))
    LOGGER.info('transcoding %s with tags (title=%s artist=%s track=%s/%s)', flac_file, tags['TITLE'], tags['ARTIST'], tags['TRACKNUMBER'], tags['TRACKTOTAL'])

    mp3_tags = {vobis_comments_lame_opts_map[k]: v for k,v in tags.items()}
    if None in mp3_tags: del mp3_tags[None]
    if 'total' in mp3_tags:
        mp3_tags['--tn'] = '%s/%s' % (tags['TRACKNUMBER'], mp3_tags.pop('total'))

    cover_file = join(dirname(flac_file), "cover.jpg")
    if os.path.isfile(cover_file): mp3_tags['--ti'] = cover_file

    lame_command_list = LAME_COMMAND.split(' ')
    lame_command_list.extend([arg for k,v in mp3_tags.items() for arg in (k,v)])
    lame_command_list.append('-')
    lame_command_list.append(mp3_file)

    flac_command = Popen(('flac -dcs %s' % flac_file).split((' ')), stdout=PIPE)
    lame_command = Popen(lame_command_list, stdin=flac_command.stdout)
    lame_command.wait()

class MetaflacNotFound(Exception):pass

def get_vobis_comment_bloc(flac_file):
    block = None
    with open(flac_file, 'rb') as flac:
        assert 'fLaC' == flac.read(4)

        last_block = False
        block_type = 0

        while not last_block and block_type is not VOBIS_COMMENT:
            last_block_and_block_type = flac.read(1)
            block_type = ord(last_block_and_block_type) & 0x07
            last_block = ord(last_block_and_block_type) & 0x80 is 0x80
            block_length, = unpack('>i', '\x00' + flac.read(3))
            block = flac.read(int(block_length))
        if block_type is not VOBIS_COMMENT: raise MetaflacNotFound()
    return block

def get_flac_tags(vobis_comment_block):
    vendor_length, = unpack('I', vobis_comment_block[0:4])
    offset = 4 + vendor_length
    user_comment_list_length, = unpack('I', vobis_comment_block[offset:offset + 4])
    offset += 4
    comments = list()
    for vobis_comment_index in range(user_comment_list_length):
        length, = unpack('I', vobis_comment_block[offset:offset + 4])
        offset += 4
        comments.append(vobis_comment_block[offset:offset + length])
        offset += length
    return none_if_missing(split_key_value_at_first_equal_and_upper_key(comment) for comment in comments)

def find_files(extension, *root_dirs):
    for root_dir in root_dirs:
        for root, _, files in os.walk(root_dir):
            for file in files:
                if file.endswith(extension): yield join(root, file)

def get_mp3_filename(mp3_target_path, flac_root_path, flac_file):
    flac_path_relative_to_root = flac_file.replace(flac_root_path, '').replace('.flac', '.mp3')
    if flac_path_relative_to_root.startswith('/'): flac_path_relative_to_root = flac_path_relative_to_root[1:]
    return join(mp3_target_path, flac_path_relative_to_root)

def process_transcoding((flac_file, flac_root_path, mp3_target_path)):
    try:
        target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file)
        if not isdir(dirname(target_mp3_file)):
            try:
                os.makedirs(dirname(target_mp3_file))
            except OSError:
                pass # other thread might have been faster
        transcode(flac_file, target_mp3_file)
    except Exception as e:
        LOGGER.error('error during the transcoding of %s : %s' % (flac_file, e))

def which(program):
    def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

def run(mp3_target_path, flac_root_path, *flac_path_list):
    flac_files = set(find_files('.flac', *flac_path_list))
    cpu_count = get_cpu_count()
    LOGGER.info('found %d cpu(s)', cpu_count)
    LOGGER.info('found %d flac files', len(flac_files))
    LOGGER.info('transcoding files with command "%s"', LAME_COMMAND)

    Pool(cpu_count).map(process_transcoding, zip(flac_files, repeat(flac_root_path), repeat(mp3_target_path)))

def split_key_value_at_first_equal_and_upper_key(string_with_equal):
    k,v = string_with_equal.split('=', 1)
    return k.upper(), v

class Usage(Exception):
    def __init__(self, msg, *args, **kwargs):
        super(Usage, self).__init__(*args, **kwargs)
        self.msg = msg

def main(argv):
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
        except getopt.error, msg:
            raise Usage(msg)

        for opt, arg in opts:
            if opt in ("-h", "--help"):
                print __doc__
                return 0

        if args:
            mp3_target_path = args.pop()
        else:
            raise Usage('no mp3 target directory given')

        if not which('lame'):
            LOGGER.fatal("Cannot find lame. Please install lame: http://lame.sourceforge.net/")
            return 3
        if not which('flac'):
            LOGGER.fatal("Cannot find flac. Please install flac: http://flac.sourceforge.net/")
            return 3

        run(mp3_target_path, getcwd(), *args)
        LOGGER.info('transcoding done, exiting normally')
        return 0
    except Usage, err:
        print >> sys.stderr, err.msg
        print >> sys.stderr, "for help use -h or --help"
        return 2

if __name__ == "__main__":
    LOGGER.setLevel(INFO)
    sys.exit(main(sys.argv))
<commit_msg>[refactor] dead code<commit_after>#!/usr/bin/env python
"""
To use this script you'll need flac and lame
Usage :
flac2mp3 [origin directories containing flac files] mp3/repository/destination
"""
import getopt
from itertools import repeat
from logging import INFO
import logging
from multiprocessing import Pool
import multiprocessing
from posix import getcwd
from struct import unpack
import sys
from genericpath import isdir
import os
from os.path import dirname, join
from subprocess import Popen, PIPE

__author__ = 'bruno thomas'

logging.basicConfig(format='%(asctime)s [%(name)s] %(levelname)s: %(message)s')
LOGGER = logging.getLogger('flac2mp3')

LAME_COMMAND = 'lame --silent -V2 --vbr-new -q0 --lowpass 19.7 --resample 44100 --add-id3v2'
VOBIS_COMMENT = 4

class none_if_missing(dict):
    def __init__(self, seq=None, **kwargs): super(none_if_missing, self).__init__(seq, **kwargs)
    def __missing__(self, _):return None

vobis_comments_lame_opts_map = none_if_missing({
   'ARTIST'     : '--ta',
   'ALBUM'      : '--tl',
   'TITLE'      : '--tt',
   'DESCRIPTION': '--tc',
   'GENRE'      : '--tg',
   'DATE'       : '--ty',
   'TRACKNUMBER': '--tn',
   'TRACKTOTAL' : 'total'
})

def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    except NotImplementedError:
        return 1

def transcode(flac_file, mp3_file):
    flac_tags = get_flac_tags(get_vobis_comment_bloc(flac_file))
    LOGGER.info('transcoding %s with tags (title=%s artist=%s track=%s/%s)', flac_file, flac_tags['TITLE'], flac_tags['ARTIST'], flac_tags['TRACKNUMBER'], flac_tags['TRACKTOTAL'])

    lame_tags = {vobis_comments_lame_opts_map[k]: v for k,v in flac_tags.items()}
    if None in lame_tags: del lame_tags[None]
    if 'total' in lame_tags:
        lame_tags['--tn'] = '%s/%s' % (flac_tags['TRACKNUMBER'], lame_tags.pop('total'))

    cover_file = join(dirname(flac_file), "cover.jpg")
    if os.path.isfile(cover_file): lame_tags['--ti'] = cover_file

    lame_command_list = LAME_COMMAND.split(' ')
    lame_command_list.extend([arg for k,v in lame_tags.items() for arg in (k,v)])
    lame_command_list.append('-')
    lame_command_list.append(mp3_file)

    flac_command = Popen(('flac -dcs %s' % flac_file).split((' ')), stdout=PIPE)
    lame_command = Popen(lame_command_list, stdin=flac_command.stdout)
    lame_command.wait()

def get_vobis_comment_bloc(flac_file):
    block = None
    with open(flac_file, 'rb') as flac:
        assert 'fLaC' == flac.read(4)

        last_block = False
        block_type = 0

        while not last_block and block_type is not VOBIS_COMMENT:
            last_block_and_block_type = flac.read(1)
            block_type = ord(last_block_and_block_type) & 0x07
            last_block = ord(last_block_and_block_type) & 0x80 is 0x80
            block_length, = unpack('>i', '\x00' + flac.read(3))
            block = flac.read(int(block_length))

        if block_type is not VOBIS_COMMENT:
            raise RuntimeError('cannot find vobis comment block in %s' % flac_file)
    return block

def get_flac_tags(vobis_comment_block):
    vendor_length, = unpack('I', vobis_comment_block[0:4])
    offset = 4 + vendor_length
    user_comment_list_length, = unpack('I', vobis_comment_block[offset:offset + 4])
    offset += 4
    comments = list()
    for vobis_comment_index in range(user_comment_list_length):
        length, = unpack('I', vobis_comment_block[offset:offset + 4])
        offset += 4
        comments.append(vobis_comment_block[offset:offset + length])
        offset += length
    return none_if_missing(split_key_value_at_first_equal_and_upper_key(comment) for comment in comments)

def find_files(extension, *root_dirs):
    for root_dir in root_dirs:
        for root, _, files in os.walk(root_dir):
            for file in files:
                if file.endswith(extension): yield join(root, file)

def get_mp3_filename(mp3_target_path, flac_root_path, flac_file):
    flac_path_relative_to_root = flac_file.replace(flac_root_path, '').replace('.flac', '.mp3')
    if flac_path_relative_to_root.startswith('/'): flac_path_relative_to_root = flac_path_relative_to_root[1:]
    return join(mp3_target_path, flac_path_relative_to_root)

def process_transcoding((flac_file, flac_root_path, mp3_target_path)):
    try:
        target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file)
        if not isdir(dirname(target_mp3_file)):
            try:
                os.makedirs(dirname(target_mp3_file))
            except OSError:
                pass # other thread might have been faster
        transcode(flac_file, target_mp3_file)
    except Exception as e:
        LOGGER.error('error during the transcoding of %s : %s' % (flac_file, e))

def which(program):
    def is_exe(fpath): return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
    fpath, fname = os.path.split(program)
    if fpath:
        if is_exe(program):
            return program
    else:
        for path in os.environ["PATH"].split(os.pathsep):
            exe_file = os.path.join(path, program)
            if is_exe(exe_file):
                return exe_file

def run(mp3_target_path, flac_root_path, *flac_path_list):
    flac_files = set(find_files('.flac', *flac_path_list))
    cpu_count = get_cpu_count()
    LOGGER.info('found %d cpu(s)', cpu_count)
    LOGGER.info('found %d flac files', len(flac_files))
    LOGGER.info('transcoding files with command "%s"', LAME_COMMAND)

    Pool(cpu_count).map(process_transcoding, zip(flac_files, repeat(flac_root_path), repeat(mp3_target_path)))

def split_key_value_at_first_equal_and_upper_key(string_with_equal):
    k,v = string_with_equal.split('=', 1)
    return k.upper(), v

class Usage(Exception):
    def __init__(self, msg, *args, **kwargs):
        super(Usage, self).__init__(*args, **kwargs)
        self.msg = msg

def main(argv):
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
        except getopt.error, msg:
            raise Usage(msg)

        for opt, arg in opts:
            if opt in ("-h", "--help"):
                print __doc__
                return 0

        if args:
            mp3_target_path = args.pop()
        else:
            raise Usage('no mp3 target directory given')

        if not which('lame'):
            LOGGER.fatal("Cannot find lame. Please install lame: http://lame.sourceforge.net/")
            return 3
        if not which('flac'):
            LOGGER.fatal("Cannot find flac. Please install flac: http://flac.sourceforge.net/")
            return 3

        run(mp3_target_path, getcwd(), *args)
        LOGGER.info('transcoding done, exiting normally')
        return 0
    except Usage, err:
        print >> sys.stderr, err.msg
        print >> sys.stderr, "for help use -h or --help"
        return 2

if __name__ == "__main__":
    LOGGER.setLevel(INFO)
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before>import os
import signal
import sys

from jupyter_helpers import notebook

def test_get_session():
    sm = notebook.SessionManager()
    session = sm.get_session()
    session.thread.stop()
    os.kill(session.process.pid, signal.SIGTERM)
    pass
<commit_msg>trying with SIGINT<commit_after>import os
import signal
import sys

from jupyter_helpers import notebook

def test_get_session():
    sm = notebook.SessionManager()
    session = sm.get_session()
    session.thread.stop()
    os.kill(session.process.pid, signal.SIGINT)
    pass
<|endoftext|>


<commit_before><commit_msg>small modifications<commit_after><|endoftext|>


<commit_before>from aiida import load_dbenv
load_dbenv()

from aiida.orm import load_node, load_workflow
from aiida.orm import Code, DataFactory

import matplotlib.pyplot as plt

StructureData = DataFactory('structure')
ParameterData = DataFactory('parameter')
ArrayData = DataFactory('array')
KpointsData = DataFactory('array.kpoints')

import numpy as np

def get_plot(data, q_path, data_name, labels=None):
    for i, freq in enumerate(data):
        plt.plot(q_path[i], freq, color='r')

    plt.axes().get_xaxis().set_ticks([])
    plt.ylabel('Frequency [THz]')
    plt.xlabel('Wave vector')
    plt.xlim([0, q_path[-1][-1]])
    plt.axhline(y=0, color='k', ls='dashed')
    plt.suptitle(data_name)

    if labels is not None:
        plt.rcParams.update({'mathtext.default':  'regular' })
        labels_e = []
        x_labels = []
        for i, freq in enumerate(q_path):
            if labels[i][0] == labels[i-1][1]:
                labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
            else:
                labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
            x_labels.append(q_path[i][0])
        x_labels.append(q_path[-1][-1])
        labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
        labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

        plt.xticks(x_labels, labels_e, rotation='horizontal')

#######################
wf = load_workflow(447)
#######################

# Phonon Band structure

bs = wf.get_result('band_structure')

plt.figure(1)
get_plot(bs.get_array('frequencies'), bs.get_array('q_path'), 'Phonon band structure', labels=bs.get_array('labels'))
plt.figure(2)
get_plot(bs.get_array('eigenvalues'), bs.get_array('q_path'), 'Eigenvalues', labels=bs.get_array('labels'))
plt.figure(3)
get_plot(bs.get_array('gamma'), bs.get_array('q_path'), 'Mode Gruneisen parameter', labels=bs.get_array('labels'))
plt.show()
exit()

plt.figure(1)
for i, freq in enumerate(bs.get_array('frequencies')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Phonon band structure')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

#plt.show()

plt.figure(2)
for i, freq in enumerate(bs.get_array('eigenvalues')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Eigenvalues')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

plt.figure(3)
for i, freq in enumerate(bs.get_array('gamma')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Gamma')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

plt.show()
<commit_msg>add gruneisen plot tool<commit_after>from aiida import load_dbenv
load_dbenv()

from aiida.orm import load_node, load_workflow
from aiida.orm import Code, DataFactory

import matplotlib.pyplot as plt

StructureData = DataFactory('structure')
ParameterData = DataFactory('parameter')
ArrayData = DataFactory('array')
KpointsData = DataFactory('array.kpoints')

import numpy as np

def get_plot(data, q_path, title=None, ylabel=None, labels=None):
    for i, freq in enumerate(data):
        plt.plot(q_path[i], freq, color='r')

    plt.axes().get_xaxis().set_ticks([])
    plt.ylabel(ylabel)
    plt.xlabel('Wave vector')
    plt.xlim([0, q_path[-1][-1]])
    plt.axhline(y=0, color='k', ls='dashed')
    plt.suptitle(title)

    if labels is not None:
        plt.rcParams.update({'mathtext.default':  'regular' })
        labels_e = []
        x_labels = []
        for i, freq in enumerate(q_path):
            if labels[i][0] == labels[i-1][1]:
                labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
            else:
                labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
            x_labels.append(q_path[i][0])
        x_labels.append(q_path[-1][-1])
        labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
        labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

        plt.xticks(x_labels, labels_e, rotation='horizontal')

#######################
wf = load_workflow(447)
#######################

# Phonon Band structure

bs = wf.get_result('band_structure')

plt.figure(1)
get_plot(bs.get_array('frequencies'), bs.get_array('q_path'),
         title='Phonon band structure',
         ylabel='Frequency [THz]',
         labels=bs.get_array('labels'))
plt.figure(2)
get_plot(bs.get_array('eigenvalues'), bs.get_array('q_path'),
         title='Eigenvalues',
         labels=bs.get_array('labels'))
plt.figure(3)
get_plot(bs.get_array('gamma'), bs.get_array('q_path'),
         title='Mode Gruneisen parameter',
         ylabel='$\Gamma$',
         labels=bs.get_array('labels'))
plt.show()
exit()

plt.figure(1)
for i, freq in enumerate(bs.get_array('frequencies')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Phonon band structure')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

#plt.show()

plt.figure(2)
for i, freq in enumerate(bs.get_array('eigenvalues')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Eigenvalues')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

plt.figure(3)
for i, freq in enumerate(bs.get_array('gamma')):
    plt.plot(bs.get_array('q_path')[i], freq, color='r')

plt.axes().get_xaxis().set_ticks([])
plt.ylabel('Frequency [THz]')
plt.xlabel('Wave vector')
plt.xlim([0, bs.get_array('q_path')[-1][-1]])
plt.axhline(y=0, color='k', ls='dashed')
plt.suptitle('Gamma')

if 'labels' in bs.get_arraynames():
    plt.rcParams.update({'mathtext.default':  'regular' })
    labels = bs.get_array('labels')

    labels_e = []
    x_labels = []
    for i, freq in enumerate(bs.get_array('q_path')):
        if labels[i][0] == labels[i-1][1]:
            labels_e.append('$'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        else:
            labels_e.append('$'+labels[i-1][1].replace('GAMMA', '\Gamma')+'/'+labels[i][0].replace('GAMMA', '\Gamma')+'$')
        x_labels.append(bs.get_array('q_path')[i][0])
    x_labels.append(bs.get_array('q_path')[-1][-1])
    labels_e.append('$'+labels[-1][1].replace('GAMMA', '\Gamma')+'$')
    labels_e[0]='$'+labels[0][0].replace('GAMMA', '\Gamma')+'$'

    plt.xticks(x_labels, labels_e, rotation='horizontal')

plt.show()
<|endoftext|>


<commit_before>from flask import current_app
from sqlalchemy.exc import IntegrityError

from zeus import auth
from zeus.config import celery, db
from zeus.constants import Status
from zeus.models import Artifact, Build, Job, PendingArtifact

from .process_artifact import process_artifact

@celery.task(max_retries=5, autoretry_for=(Exception,), acks_late=True, time_limit=60)
def process_pending_artifact(pending_artifact_id, **kwargs):
    pending_artifact = PendingArtifact.query.unrestricted_unsafe().get(
        pending_artifact_id
    )
    if pending_artifact is None:
        current_app.logger.error("PendingArtifact %s not found", pending_artifact_id)
        return

    db.session.delete(pending_artifact)
    db.session.flush()

    auth.set_current_tenant(
        auth.RepositoryTenant(repository_id=pending_artifact.repository_id)
    )

    build = Build.query.filter(
        Build.repository_id == pending_artifact.repository_id,
        Build.provider == pending_artifact.provider,
        Build.external_id == pending_artifact.external_build_id,
    ).first()
    if not build:
        raise NotImplementedError

    job = Job.query.filter(
        Job.repository_id == pending_artifact.repository_id,
        Job.build_id == build.id,
        Job.provider == pending_artifact.provider,
        Job.external_id == pending_artifact.external_job_id,
    ).first()
    if not job:
        raise NotImplementedError

    artifact = Artifact(
        job_id=job.id,
        repository_id=pending_artifact.repository_id,
        name=pending_artifact.name,
        status=Status.queued,
    )

    try:
        db.session.add(artifact)
        db.session.flush()
    except IntegrityError:
        current_app.logger.error(
            "Skipping pending artifact processing (%s) - duplicate key",
            pending_artifact_id,
        )
        # XXX(dcramer): this is more of an error but we make an assumption
        # that this happens because it was already sent
        db.session.rollback()
        db.session.delete(pending_artifact)
        db.session.commit()
        return

    artifact.file.save(
        pending_artifact.file,
        # XXX(dcramer): we reference the same file, so it lives in the old path
        # "{0}/{1}/{2}_{3}".format(
        #     job.id.hex[:4], job.id.hex[4:], artifact.id.hex, artifact.name
        # ),
    )
    db.session.add(artifact)

    if job.status == Status.finished:
        job.status = Status.collecting_results
        db.session.add(job)

    db.session.commit()

    process_artifact.delay(artifact_id=artifact.id)
<commit_msg>fix: Dont reset job from aborted<commit_after>from flask import current_app
from sqlalchemy.exc import IntegrityError

from zeus import auth
from zeus.config import celery, db
from zeus.constants import Result, Status
from zeus.models import Artifact, Build, Job, PendingArtifact

from .process_artifact import process_artifact

@celery.task(max_retries=5, autoretry_for=(Exception,), acks_late=True, time_limit=60)
def process_pending_artifact(pending_artifact_id, **kwargs):
    pending_artifact = PendingArtifact.query.unrestricted_unsafe().get(
        pending_artifact_id
    )
    if pending_artifact is None:
        current_app.logger.error("PendingArtifact %s not found", pending_artifact_id)
        return

    db.session.delete(pending_artifact)
    db.session.flush()

    auth.set_current_tenant(
        auth.RepositoryTenant(repository_id=pending_artifact.repository_id)
    )

    build = Build.query.filter(
        Build.repository_id == pending_artifact.repository_id,
        Build.provider == pending_artifact.provider,
        Build.external_id == pending_artifact.external_build_id,
    ).first()
    if not build:
        raise NotImplementedError

    job = Job.query.filter(
        Job.repository_id == pending_artifact.repository_id,
        Job.build_id == build.id,
        Job.provider == pending_artifact.provider,
        Job.external_id == pending_artifact.external_job_id,
    ).first()
    if not job:
        raise NotImplementedError

    artifact = Artifact(
        job_id=job.id,
        repository_id=pending_artifact.repository_id,
        name=pending_artifact.name,
        status=Status.queued,
    )

    try:
        db.session.add(artifact)
        db.session.flush()
    except IntegrityError:
        current_app.logger.error(
            "Skipping pending artifact processing (%s) - duplicate key",
            pending_artifact_id,
        )
        # XXX(dcramer): this is more of an error but we make an assumption
        # that this happens because it was already sent
        db.session.rollback()
        db.session.delete(pending_artifact)
        db.session.commit()
        return

    artifact.file.save(
        pending_artifact.file,
        # XXX(dcramer): we reference the same file, so it lives in the old path
        # "{0}/{1}/{2}_{3}".format(
        #     job.id.hex[:4], job.id.hex[4:], artifact.id.hex, artifact.name
        # ),
    )
    db.session.add(artifact)

    if job.status == Status.finished and job.result != Result.aborted:
        job.status = Status.collecting_results
        db.session.add(job)

    db.session.commit()

    process_artifact.delay(artifact_id=artifact.id)
<|endoftext|>


<commit_before># aml - Machine learning library.
# Neural network module.
# Author: Julian Ailan
#===================================

"""Provides an abstraction to generate neural networks."""

import numpy as np
import threshold as th
import perceptron as p

class NeuralNetwork(object):
    """All against all neural network architecture."""

    def __init__(self, number_of_inputs=2, number_of_outputs=1,
                 number_of_hidden_layers=1, units_per_hidden_layer=2,
                 learning_factor=1, epochs=50, activation_function="sigmod"):

        self.number_of_inputs = number_of_inputs
        self.number_of_outputs = number_of_outputs

        self.number_of_hidden_layers = number_of_hidden_layers
        self.units_per_hidden_layer = units_per_hidden_layer
        self.learning_factor = learning_factor
        self.epochs = epochs

        if activation_function == "unitstep":
            self.activation_function = th.UnitStep()
        else:
            self.activation_function = th.Sigmoid()

        self.synapses_weights = self.init_weights(number_of_inputs,
                                                  number_of_outputs,
                                                  number_of_hidden_layers,
                                                  units_per_hidden_layer)
# TODO: Generate an array of different perceptron objects more efficiently.
# TODO: Set the weights vector in each new instance. ITS VARIABLE! Initialize
#       weights and perceptrons in the same method!
        total_number_of_units = ((number_of_hidden_layers *
                                  units_per_hidden_layer) + number_of_outputs)
        self.hidden_layers_units = []
        for unit in xrange(total_number_of_units):
            self.hidden_layers_units.append(p.Perceptron())

        print len(self.synapses_weights), len(self.hidden_layers_units)

    def init_weights(self, number_of_inputs, number_of_outputs,
                     number_of_hidden_layers, units_per_hidden_layer):
        """Initializes the weights of the network synapses.

        Args:
            number_of_inputs: Number of input weights to the first hidden layer.
            number_of_outputs: Number of output weights from the last hidden
                               layer.
            number_of_hidden_layers: Number of layers of hidden units.
            units_per_hidden_layer: Number of processing units per hidden layer.

        Returns:
            w: Initial neural network synapses weights.
        """

        number_of_hidden_weights = (pow(units_per_hidden_layer, 2) *
                                    (number_of_hidden_layers - 1))

        number_of_input_weights = number_of_inputs * units_per_hidden_layer

        number_of_output_weights = number_of_outputs * units_per_hidden_layer

        total_number_of_weights = (number_of_input_weights +
                                   number_of_hidden_weights +
                                   number_of_output_weights)

        return 2 * np.random.rand(total_number_of_weights) - 1

    def feedforward(self, input_sample):
        """Computes the output of using 'input_sample' as network input.

        Args:
            input_sample: training sample provided.

        Returns:
            output: output value of the feedforward procedure.
        """
        output = []
# TODO: This section must be completed only when Perceptrons are initialized
#       with their respective weights.
        # Perform the dot product of the inputs with the first hidden layer
        #first_layer_units = self.hidden_layers_nodes[:units_per_hidden_layer]
        #for first_hidden_layer_unit in first_hidden_layer_units:
        #    output.append(first_hidden_layer_unit.output(input_sample))

# TODO: Erase this section when done.
        # Perform the dot products of the inputs & outputs of hidden layers
        #start = units_per_hidden_layer
        #end = (units_per_hidden_layer - 1) * number_of_hidden_layers
        #for hidden_unit in self.hidden_layers_nodes[start:end]:
        #    pass
        # Perform the dot product of the last layer outputs with the output layer

    def train(self, training_sample, expected_output):
        """Computes the components of the weights vector 'w' across all layers.

        Args:
            training_sample: Sample M-dimensional array.
            expected_output: Array of labels that represent the sample's class.

        Returns:
            w: Weights vector, containing the weights of all the synapses.
        """
        predicted_output = self.feedforward(training_sample)
<commit_msg>TODO: Connect layers, not units.<commit_after># aml - Machine learning library.
# Neural network module.
# Author: Julian Ailan
#===================================

"""Provides an abstraction to generate neural networks."""

import numpy as np
import threshold as th
import perceptron as p

class NeuralNetwork(object):
    """All against all neural network architecture."""

    def __init__(self, number_of_inputs=2, number_of_outputs=1,
                 number_of_hidden_layers=1, units_per_hidden_layer=2,
                 learning_factor=1, epochs=50, activation_function="sigmod"):

        self.number_of_inputs = number_of_inputs
        self.number_of_outputs = number_of_outputs

        self.number_of_hidden_layers = number_of_hidden_layers
        self.units_per_hidden_layer = units_per_hidden_layer
        self.learning_factor = learning_factor
        self.epochs = epochs

        if activation_function == "unitstep":
            self.activation_function = th.UnitStep()
        else:
            self.activation_function = th.Sigmoid()

        self.synapses_weights = self.init_weights(number_of_inputs,
                                                  number_of_outputs,
                                                  number_of_hidden_layers,
                                                  units_per_hidden_layer)
# TODO: DONT GENERATE ALL AT ONCE, MAKE LAYERS AND CONNECT LAYERS!

# TODO: Generate an array of different perceptron objects more efficiently.
# TODO: Set the weights vector in each new instance. ITS VARIABLE! Initialize
#       weights and perceptrons in the same method!
        total_number_of_units = ((number_of_hidden_layers *
                                  units_per_hidden_layer) + number_of_outputs)
        self.hidden_layers_units = []
        for unit in xrange(total_number_of_units):
            self.hidden_layers_units.append(p.Perceptron())

        print len(self.synapses_weights), len(self.hidden_layers_units)

    def init_weights(self, number_of_inputs, number_of_outputs,
                     number_of_hidden_layers, units_per_hidden_layer):
        """Initializes the weights of the network synapses.

        Args:
            number_of_inputs: Number of input weights to the first hidden layer.
            number_of_outputs: Number of output weights from the last hidden
                               layer.
            number_of_hidden_layers: Number of layers of hidden units.
            units_per_hidden_layer: Number of processing units per hidden layer.

        Returns:
            w: Initial neural network synapses weights.
        """

        number_of_hidden_weights = (pow(units_per_hidden_layer, 2) *
                                    (number_of_hidden_layers - 1))

        number_of_input_weights = number_of_inputs * units_per_hidden_layer

        number_of_output_weights = number_of_outputs * units_per_hidden_layer

        total_number_of_weights = (number_of_input_weights +
                                   number_of_hidden_weights +
                                   number_of_output_weights)

        return 2 * np.random.rand(total_number_of_weights) - 1

    def feedforward(self, input_sample):
        """Computes the output of using 'input_sample' as network input.

        Args:
            input_sample: training sample provided.

        Returns:
            output: output value of the feedforward procedure.
        """
        output = []
# TODO: This section must be completed only when Perceptrons are initialized
#       with their respective weights.
        # Perform the dot product of the inputs with the first hidden layer
        #first_layer_units = self.hidden_layers_nodes[:units_per_hidden_layer]
        #for first_hidden_layer_unit in first_hidden_layer_units:
        #    output.append(first_hidden_layer_unit.output(input_sample))

# TODO: Erase this section when done.
        # Perform the dot products of the inputs & outputs of hidden layers
        #start = units_per_hidden_layer
        #end = (units_per_hidden_layer - 1) * number_of_hidden_layers
        #for hidden_unit in self.hidden_layers_nodes[start:end]:
        #    pass
        # Perform the dot product of the last layer outputs with the output layer

    def train(self, training_sample, expected_output):
        """Computes the components of the weights vector 'w' across all layers.

        Args:
            training_sample: Sample M-dimensional array.
            expected_output: Array of labels that represent the sample's class.

        Returns:
            w: Weights vector, containing the weights of all the synapses.
        """
        predicted_output = self.feedforward(training_sample)
<|endoftext|>