<commit_before>"""
    muffin-session description.

"""

# Package information
# ===================

import asyncio
import base64
import functools
import time

import ujson as json

from muffin import HTTPFound
from muffin.plugins import BasePlugin
from muffin.utils import create_signature, check_signature, to_coroutine

__version__ = "0.0.2"
__project__ = "muffin-session"
__author__ = "Kirill Klenov <horneds@gmail.com>"
__license__ = "MIT"

FUNC = lambda x: x # noqa

class Plugin(BasePlugin):

    """ Support sessions. """

    name = 'session'
    defaults = {
        'default_user_checker': lambda x: x,
        'login_url': '/login',
        'secret': 'InsecureSecret',
    }

    def setup(self, app):
        """ Initialize the application. """
        super().setup(app)

        if self.options['secret'] == 'InsecureSecret':
            app.logger.warn('Use insecure secret key. Change AUTH_SECRET option in configuration.')

        self._user_loader = asyncio.coroutine(lambda id_: id_)

    @asyncio.coroutine
    def middleware_factory(self, app, handler):
        """ Provide session middleware. """
        @asyncio.coroutine
        def middleware(request):
            """ Load a session from users cookies. """
            request.session = Session(self.options['secret'])
            request.session.load(request.cookies)
            app.logger.debug('Started: %s', request.session)
            response = yield from handler(request)
            app.logger.debug('Ended: %s', request.session)
            request.session.save(response.set_cookie)
            return response
        return middleware

    def user_loader(self, func):
        """ Register a function as user loader. """
        self._user_loader = to_coroutine(func)
        return self._user_loader

    @asyncio.coroutine
    def load_user(self, request):
        """ Load user from request. """
        _id = request.session.get('id')
        if not _id:
            return None
        request.user = yield from self._user_loader(_id)
        return request.user

    @asyncio.coroutine
    def check_user(self, request, func=FUNC, location=None, **kwargs):
        """ Check for user is logged and pass func. """
        user = yield from self.load_user(request)
        func = func or self.options.default_user_checker
        if not func(user):
            raise HTTPFound(location or self.options.login_url, **kwargs)
        return user

    def user_pass(self, func=None, location=None, **rkwargs):
        def wrapper(view):
            view = to_coroutine(view)

            @asyncio.coroutine
            @functools.wraps(view)
            def handler(request, *args, **kwargs):
                yield from self.check_user(request, func, location, **rkwargs)
                return (yield from view(request, *args, **kwargs))
            return handler

        return wrapper

    @staticmethod
    @asyncio.coroutine
    def login(request, id):
        """ Login an user by ID. """
        request.session['id'] = id

    @staticmethod
    @asyncio.coroutine
    def logout(request):
        """ Logout an user. """
        del request.session['id']

class Session(dict):

    encoding = 'utf-8'

    def __init__(self, secret, key='session.id', **params):
        self.secret = secret.encode(self.encoding)
        self.key = key
        self.params = params
        self.store = {}

    def save(self, set_cookie):
        if set(self.store.items()) ^ set(self.items()):
            value = dict(self.items())
            value = json.dumps(value)
            value = self.encrypt(value)
            if not isinstance(value, str):
                value = value.encode(self.encoding)
            set_cookie(self.key, value, **self.params)

    def load(self, cookies, **kwargs):
        value = cookies.get(self.key, None)
        if value is None:
            return False
        value = self.decrypt(value)
        if not value:
            return False

        data = json.loads(value)
        if not isinstance(data, dict):
            return False

        self.store = data
        self.update(self.store)

    def encrypt(self, value):
        timestamp = str(int(time.time()))
        value = base64.b64encode(value.encode(self.encoding))
        signature = create_signature(self.secret, value + timestamp.encode(),
                                     encoding=self.encoding)
        return "|".join([value.decode(self.encoding), timestamp, signature])

    def decrypt(self, value):
        value, timestamp, signature = value.split("|")
        if check_signature(signature, self.secret, value + timestamp, encoding=self.encoding):
            return base64.b64decode(value).decode(self.encoding)
        return 'null'
<commit_msg>Fix insecure message<commit_after>"""
    muffin-session description.

"""

# Package information
# ===================

import asyncio
import base64
import functools
import time

import ujson as json

from muffin import HTTPFound
from muffin.plugins import BasePlugin
from muffin.utils import create_signature, check_signature, to_coroutine

__version__ = "0.0.2"
__project__ = "muffin-session"
__author__ = "Kirill Klenov <horneds@gmail.com>"
__license__ = "MIT"

FUNC = lambda x: x # noqa

class Plugin(BasePlugin):

    """ Support sessions. """

    name = 'session'
    defaults = {
        'default_user_checker': lambda x: x,
        'login_url': '/login',
        'secret': 'InsecureSecret',
    }

    def setup(self, app):
        """ Initialize the application. """
        super().setup(app)

        if self.options['secret'] == 'InsecureSecret':
            app.logger.warn(
                'Use insecure secret key. Change SESSION_SECRET option in configuration.')

        self._user_loader = asyncio.coroutine(lambda id_: id_)

    @asyncio.coroutine
    def middleware_factory(self, app, handler):
        """ Provide session middleware. """
        @asyncio.coroutine
        def middleware(request):
            """ Load a session from users cookies. """
            request.session = Session(self.options['secret'])
            request.session.load(request.cookies)
            app.logger.debug('Started: %s', request.session)
            response = yield from handler(request)
            app.logger.debug('Ended: %s', request.session)
            request.session.save(response.set_cookie)
            return response
        return middleware

    def user_loader(self, func):
        """ Register a function as user loader. """
        self._user_loader = to_coroutine(func)
        return self._user_loader

    @asyncio.coroutine
    def load_user(self, request):
        """ Load user from request. """
        _id = request.session.get('id')
        if not _id:
            return None
        request.user = yield from self._user_loader(_id)
        return request.user

    @asyncio.coroutine
    def check_user(self, request, func=FUNC, location=None, **kwargs):
        """ Check for user is logged and pass func. """
        user = yield from self.load_user(request)
        func = func or self.options.default_user_checker
        if not func(user):
            raise HTTPFound(location or self.options.login_url, **kwargs)
        return user

    def user_pass(self, func=None, location=None, **rkwargs):
        def wrapper(view):
            view = to_coroutine(view)

            @asyncio.coroutine
            @functools.wraps(view)
            def handler(request, *args, **kwargs):
                yield from self.check_user(request, func, location, **rkwargs)
                return (yield from view(request, *args, **kwargs))
            return handler

        return wrapper

    @staticmethod
    @asyncio.coroutine
    def login(request, id):
        """ Login an user by ID. """
        request.session['id'] = id

    @staticmethod
    @asyncio.coroutine
    def logout(request):
        """ Logout an user. """
        del request.session['id']

class Session(dict):

    encoding = 'utf-8'

    def __init__(self, secret, key='session.id', **params):
        self.secret = secret.encode(self.encoding)
        self.key = key
        self.params = params
        self.store = {}

    def save(self, set_cookie):
        if set(self.store.items()) ^ set(self.items()):
            value = dict(self.items())
            value = json.dumps(value)
            value = self.encrypt(value)
            if not isinstance(value, str):
                value = value.encode(self.encoding)
            set_cookie(self.key, value, **self.params)

    def load(self, cookies, **kwargs):
        value = cookies.get(self.key, None)
        if value is None:
            return False
        value = self.decrypt(value)
        if not value:
            return False

        data = json.loads(value)
        if not isinstance(data, dict):
            return False

        self.store = data
        self.update(self.store)

    def encrypt(self, value):
        timestamp = str(int(time.time()))
        value = base64.b64encode(value.encode(self.encoding))
        signature = create_signature(self.secret, value + timestamp.encode(),
                                     encoding=self.encoding)
        return "|".join([value.decode(self.encoding), timestamp, signature])

    def decrypt(self, value):
        value, timestamp, signature = value.split("|")
        if check_signature(signature, self.secret, value + timestamp, encoding=self.encoding):
            return base64.b64decode(value).decode(self.encoding)
        return 'null'
<|endoftext|>


<commit_before>"""
Test SBThread APIs.
"""

import os, time
import unittest2
import lldb
from lldbutil import get_stopped_thread, get_caller_symbol
from lldbtest import *

class ThreadAPITestCase(TestBase):

    mydir = os.path.join("python_api", "thread")

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test
    def test_get_stop_description_with_dsym(self):
        """Test Python SBThread.GetStopDescription() API."""
        self.buildDsym()
        self.get_stop_description()

    @python_api_test
    def test_get_stop_description_with_dwarf(self):
        """Test Python SBThread.GetStopDescription() API."""
        self.buildDwarf()
        self.get_stop_description()

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test
    def test_step_out_of_malloc_into_function_b_with_dsym(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        # We build a different executable than the default buildDsym() does.
        d = {'CXX_SOURCES': 'main2.cpp'}
        self.buildDsym(dictionary=d)
        self.setTearDownCleanup(dictionary=d)
        self.step_out_of_malloc_into_function_b()

    @python_api_test
    def test_step_out_of_malloc_into_function_b_with_dwarf(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        # We build a different executable than the default buildDwarf() does.
        d = {'CXX_SOURCES': 'main2.cpp'}
        self.buildDwarf(dictionary=d)
        self.setTearDownCleanup(dictionary=d)
        self.step_out_of_malloc_into_function_b()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number("main.cpp", "// Set break point at this line and check variable 'my_char'.")
        # Find the line number within main2.cpp for step_out_of_malloc_into_function_b().
        self.line2 = line_number("main2.cpp", "// thread step-out of malloc into function b.")

    def get_stop_description(self):
        """Test Python SBProcess.ReadMemory() API."""
        exe = os.path.join(os.getcwd(), "a.out")
        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target.IsValid(), VALID_TARGET)

        breakpoint = target.BreakpointCreateByLocation("main.cpp", self.line)
        self.assertTrue(breakpoint.IsValid(), VALID_BREAKPOINT)
        #self.runCmd("breakpoint list")

        # Launch the process, and do not stop at the entry point.
        error = lldb.SBError()
        self.process = target.Launch (self.dbg.GetListener(), None, None, os.ctermid(), os.ctermid(), os.ctermid(), None, 0, False, error)

        thread = get_stopped_thread(self.process, lldb.eStopReasonBreakpoint)
        self.assertTrue(thread != None, "There should be a thread stopped due to breakpoint")
        #self.runCmd("process status")

        # Due to the typemap magic (see lldb.swig), we pass in an (int)length to GetStopDescription
        # and expect to get a Python string as the result object!
        # The 100 is just an arbitrary number specifying the buffer size.
        stop_description = thread.GetStopDescription(100)
        self.expect(stop_description, exe=False,
            startstr = 'breakpoint')

    def step_out_of_malloc_into_function_b(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        exe = os.path.join(os.getcwd(), "a.out")
        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target.IsValid(), VALID_TARGET)

        breakpoint = target.BreakpointCreateByName('malloc')
        self.assertTrue(breakpoint.IsValid(), VALID_BREAKPOINT)
        self.runCmd("breakpoint list")

        # Launch the process, and do not stop at the entry point.
        error = lldb.SBError()
        self.process = target.Launch (self.dbg.GetListener(), None, None, os.ctermid(), os.ctermid(), os.ctermid(), None, 0, False, error)

        thread = get_stopped_thread(self.process, lldb.eStopReasonBreakpoint)
        self.assertTrue(thread != None, "There should be a thread stopped due to breakpoint")
        self.runCmd("process status")
        symbol = get_caller_symbol(thread)
        caller = symbol.split('(')[0]

        while caller != "b":
            #self.runCmd("thread backtrace")
            #self.runCmd("process status")
            self.process.Continue()
            thread = get_stopped_thread(self.process, lldb.eStopReasonBreakpoint)
            symbol = get_caller_symbol(thread)
            caller = symbol.split('(')[0]
            self.assertTrue(thread != None, "There should be a thread stopped due to breakpoint")

        thread.StepOut()
        self.assertTrue(thread.GetFrameAtIndex(0).GetLineEntry().GetLine() == self.line2,
                        "step out of malloc into function b is successful")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<commit_msg>Test cleanup.  Check for the full caller symbol of malloc -- b(int).<commit_after>"""
Test SBThread APIs.
"""

import os, time
import unittest2
import lldb
from lldbutil import get_stopped_thread, get_caller_symbol
from lldbtest import *

class ThreadAPITestCase(TestBase):

    mydir = os.path.join("python_api", "thread")

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test
    def test_get_stop_description_with_dsym(self):
        """Test Python SBThread.GetStopDescription() API."""
        self.buildDsym()
        self.get_stop_description()

    @python_api_test
    def test_get_stop_description_with_dwarf(self):
        """Test Python SBThread.GetStopDescription() API."""
        self.buildDwarf()
        self.get_stop_description()

    @unittest2.skipUnless(sys.platform.startswith("darwin"), "requires Darwin")
    @python_api_test
    def test_step_out_of_malloc_into_function_b_with_dsym(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        # We build a different executable than the default buildDsym() does.
        d = {'CXX_SOURCES': 'main2.cpp'}
        self.buildDsym(dictionary=d)
        self.setTearDownCleanup(dictionary=d)
        self.step_out_of_malloc_into_function_b()

    @python_api_test
    def test_step_out_of_malloc_into_function_b_with_dwarf(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        # We build a different executable than the default buildDwarf() does.
        d = {'CXX_SOURCES': 'main2.cpp'}
        self.buildDwarf(dictionary=d)
        self.setTearDownCleanup(dictionary=d)
        self.step_out_of_malloc_into_function_b()

    def setUp(self):
        # Call super's setUp().
        TestBase.setUp(self)
        # Find the line number to break inside main().
        self.line = line_number("main.cpp", "// Set break point at this line and check variable 'my_char'.")
        # Find the line number within main2.cpp for step_out_of_malloc_into_function_b().
        self.line2 = line_number("main2.cpp", "// thread step-out of malloc into function b.")

    def get_stop_description(self):
        """Test Python SBProcess.ReadMemory() API."""
        exe = os.path.join(os.getcwd(), "a.out")
        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target.IsValid(), VALID_TARGET)

        breakpoint = target.BreakpointCreateByLocation("main.cpp", self.line)
        self.assertTrue(breakpoint.IsValid(), VALID_BREAKPOINT)
        #self.runCmd("breakpoint list")

        # Launch the process, and do not stop at the entry point.
        error = lldb.SBError()
        self.process = target.Launch (self.dbg.GetListener(), None, None, os.ctermid(), os.ctermid(), os.ctermid(), None, 0, False, error)

        thread = get_stopped_thread(self.process, lldb.eStopReasonBreakpoint)
        self.assertTrue(thread != None, "There should be a thread stopped due to breakpoint")
        #self.runCmd("process status")

        # Due to the typemap magic (see lldb.swig), we pass in an (int)length to GetStopDescription
        # and expect to get a Python string as the result object!
        # The 100 is just an arbitrary number specifying the buffer size.
        stop_description = thread.GetStopDescription(100)
        self.expect(stop_description, exe=False,
            startstr = 'breakpoint')

    def step_out_of_malloc_into_function_b(self):
        """Test Python SBThread.StepOut() API to step out of a malloc call where the call site is at function b()."""
        exe = os.path.join(os.getcwd(), "a.out")
        self.runCmd("file " + exe, CURRENT_EXECUTABLE_SET)

        target = self.dbg.CreateTarget(exe)
        self.assertTrue(target.IsValid(), VALID_TARGET)

        breakpoint = target.BreakpointCreateByName('malloc')
        self.assertTrue(breakpoint.IsValid(), VALID_BREAKPOINT)
        self.runCmd("breakpoint list")

        # Launch the process, and do not stop at the entry point.
        error = lldb.SBError()
        self.process = target.Launch (self.dbg.GetListener(), None, None, os.ctermid(), os.ctermid(), os.ctermid(), None, 0, False, error)

        while True:
            thread = get_stopped_thread(self.process, lldb.eStopReasonBreakpoint)
            self.assertTrue(thread != None, "There should be a thread stopped due to breakpoint")
            caller_symbol = get_caller_symbol(thread)
            #print "caller symbol of malloc:", caller_symbol
            if not caller_symbol:
                self.fail("Test failed: could not locate the caller symbol of malloc")
            if caller_symbol == "b(int)":
                break
            #self.runCmd("thread backtrace")
            #self.runCmd("process status")
            self.process.Continue()

        thread.StepOut()
        self.runCmd("thread backtrace")
        #self.runCmd("process status")
        self.assertTrue(thread.GetFrameAtIndex(0).GetLineEntry().GetLine() == self.line2,
                        "step out of malloc into function b is successful")

if __name__ == '__main__':
    import atexit
    lldb.SBDebugger.Initialize()
    atexit.register(lambda: lldb.SBDebugger.Terminate())
    unittest2.main()
<|endoftext|>


<commit_before>import hashlib
import autopep8

from IPython.utils.py3compat import str_to_bytes

def is_grade(cell):
    """Returns True if the cell is a grade cell."""
    if 'nbgrader' not in cell.metadata:
        return False
    return cell.metadata['nbgrader'].get('grade', False)

def is_solution(cell):
    """Returns True if the cell is a solution cell."""
    if 'nbgrader' not in cell.metadata:
        return False
    return cell.metadata['nbgrader'].get('solution', False)

def determine_grade(cell):
    if not is_grade(cell):
        raise ValueError("cell is not a grade cell")

    max_points = float(cell.metadata['nbgrader']['points'])
    if cell.cell_type == 'code':
        for output in cell.outputs:
            if output.output_type == 'error':
                return 0, max_points
        return max_points, max_points

    else:
        return None, max_points

def compute_checksum(cell):
    m = hashlib.md5()

    # fix minor whitespace issues that might have been added and then
    # add cell contents
    m.update(str_to_bytes(autopep8.fix_code(cell.source).rstrip()))

    # add the cell type
    m.update(str_to_bytes(cell.cell_type))

    # add whether it's a grade cell and/or solution cell
    m.update(str_to_bytes(str(is_grade(cell))))
    m.update(str_to_bytes(str(is_solution(cell))))

    # include the grade id and the number of points that the cell is worth
    if is_grade(cell):
        m.update(str_to_bytes(str(float(cell.metadata.nbgrader['points']))))
        m.update(str_to_bytes(cell.metadata.nbgrader['grade_id']))

    return m.hexdigest()
<commit_msg>Only use autopep8 on code if it's a code cell<commit_after>import hashlib
import autopep8

from IPython.utils.py3compat import str_to_bytes

def is_grade(cell):
    """Returns True if the cell is a grade cell."""
    if 'nbgrader' not in cell.metadata:
        return False
    return cell.metadata['nbgrader'].get('grade', False)

def is_solution(cell):
    """Returns True if the cell is a solution cell."""
    if 'nbgrader' not in cell.metadata:
        return False
    return cell.metadata['nbgrader'].get('solution', False)

def determine_grade(cell):
    if not is_grade(cell):
        raise ValueError("cell is not a grade cell")

    max_points = float(cell.metadata['nbgrader']['points'])
    if cell.cell_type == 'code':
        for output in cell.outputs:
            if output.output_type == 'error':
                return 0, max_points
        return max_points, max_points

    else:
        return None, max_points

def compute_checksum(cell):
    m = hashlib.md5()

    # if it's a code cell, then fix minor whitespace issues that might have been
    # added and then add cell contents; otherwise just add cell contents
    if cell.cell_type == "code":
        m.update(str_to_bytes(autopep8.fix_code(cell.source).rstrip()))
    else:
        m.update(str_to_bytes(cell.source.strip()))

    # add the cell type
    m.update(str_to_bytes(cell.cell_type))

    # add whether it's a grade cell and/or solution cell
    m.update(str_to_bytes(str(is_grade(cell))))
    m.update(str_to_bytes(str(is_solution(cell))))

    # include the grade id and the number of points that the cell is worth
    if is_grade(cell):
        m.update(str_to_bytes(str(float(cell.metadata.nbgrader['points']))))
        m.update(str_to_bytes(cell.metadata.nbgrader['grade_id']))

    return m.hexdigest()
<|endoftext|>


<commit_before>
"""This module exposes utilities for parsing literal nbt strings.

Exported items:
    parse_nbt      -- Helper function that parses nbt tags
    InvalidLiteral -- Exception raised when parsing invalid nbt literals
"""

__all__ = ['parse_nbt', 'InvalidLiteral']

import re
import json
from collections import namedtuple

from .tag import *

# Token definition

TOKENS = {
    'QUOTED_STRING': r'""|".*?[^\\]"',
    'NUMBER': r'[+-]?(?:[0-9]*?\.[0-9]+|[0-9]+\.[0-9]*?|[0-9]+)[bslfdBSLFD]?',
    'STRING': r'[a-zA-Z0-9._+-]+',
    'COMPOUND': r'\{',
    'CLOSE_COMPOUND': r'\}',
    'BYTE_ARRAY': r'\[B;',
    'INT_ARRAY': r'\[I;',
    'LONG_ARRAY': r'\[L;',
    'LIST': r'\[',
    'CLOSE_BRACKET': r'\]',
    'COLON': r':',
    'COMMA': r',',
    'INVALID': r'.+?',
}

# Build the regex

TOKENS_REGEX = re.compile(
    '|'.join(f'\s*(?P<{key}>{value})\s*' for key, value in TOKENS.items())
)

# Associate number suffixes to tag types

NUMBER_SUFFIXES = {'b': Byte, 's': Short, 'l': Long, 'f': Float, 'd': Double}

def parse_nbt(literal):
    """Parse a literal nbt string and return the resulting tag."""
    parser = NbtParser(tokenize(literal))
    tag = parser.parse()

    cursor = parser.token_span[1]
    leftover = literal[cursor:]
    if leftover.strip():
        parser.token_span = cursor, cursor + len(leftover)
        raise parser.error(f'Expected end of string but got {leftover!r}')

    return tag

class InvalidLiteral(ValueError):
    """Exception raised when parsing invalid nbt literals.

    The exception must be instantiated with two parameters. The first
    one needs to be a tuple representing the location of the error in
    the nbt string (start_index, end_index). The second argument is the
    actual error message.
    """

    def __str__(self):
        return f'{self.args[1]} at position {self.args[0][0]}'

Token = namedtuple('Token', ['type', 'value', 'span'])

def tokenize(string):
    """Match and yield all the tokens of the input string."""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())

class NbtParser:
    """Nbt literal parser.

    The parser needs to be instantiated with a token stream as argument.
    Using the `parse` method will return the corresponding nbt tag.

    The parser will raise an InvalidLiteral exception if it encounters
    an invalid nbt literal while parsing.
    """

    def __init__(self, token_stream):
        self.token_stream = iter(token_stream)
        self.current_token = None
        self.token_span = (0, 0)

        self.next()

    def error(self, message):
        """Create an InvalidLiteral using the current token position."""
        return InvalidLiteral(self.token_span, message)

    def next(self):
        """Move to the next token in the token stream."""
        self.current_token = next(self.token_stream, None)
        if self.current_token is None:
            self.token_span = self.token_span[1], self.token_span[1]
            raise self.error('Unexpected end of input')
        self.token_span = self.current_token.span
        return self

    def parse(self):
        """Parse and return an nbt literal from the token stream."""
        token_type = self.current_token.type.lower()
        handler = getattr(self, f'parse_{token_type}', None)
        if handler is None:
            raise self.error(f'Invalid literal {self.current_token.value!r}')
        return handler()

    def parse_quoted_string(self):
        """Parse a quoted string from the token stream."""
        return String(self.unquote_string(self.current_token.value))

    def parse_number(self):
        """Parse a number from the token stream."""
        value = self.current_token.value
        suffix = value[-1].lower()
        if suffix in NUMBER_SUFFIXES:
            return NUMBER_SUFFIXES[suffix](value[:-1])
        else:
            return Int(value)

    def parse_string(self):
        """Parse a regular unquoted string from the token stream."""
        return String(self.current_token.value)

    def collect_tokens_until(self, token_type):
        """Yield the item tokens in a comma-separated tag collection."""
        self.next()
        if self.current_token.type == token_type:
            return

        while True:
            yield self.current_token

            self.next()
            if self.current_token.type == token_type:
                return

            if self.current_token.type != 'COMMA':
                raise self.error(f'Expected comma but got '
                                 f'{self.current_token.value!r}')
            self.next()

    def parse_compound(self):
        """Parse a compound from the token stream."""
        compound_tag = Compound()

        for token in self.collect_tokens_until('CLOSE_COMPOUND'):
            item_key = token.value
            if token.type not in ('STRING', 'QUOTED_STRING'):
                raise self.error(f'Expected compound key but got {item_key!r}')

            if token.type == 'QUOTED_STRING':
                item_key = self.unquote_string(item_key)

            if self.next().current_token.type != 'COLON':
                raise self.error(f'Expected colon but got '
                                 f'{self.current_token.value!r}')
            self.next()
            compound_tag[item_key] = self.parse()
        return compound_tag

    def array_items(self, number_type, *, number_suffix=''):
        """Parse and yield array items from the token stream."""
        for token in self.collect_tokens_until('CLOSE_BRACKET'):
            is_number = token.type == 'NUMBER'
            value = token.value.lower()
            if not (is_number and value.endswith(number_suffix)):
                raise self.error(f'Invalid {number_type} array element '
                                 f'{token.value!r}')
            yield int(value.replace(number_suffix, ''))

    def parse_byte_array(self):
        """Parse a byte array from the token stream."""
        return ByteArray(list(self.array_items('byte', number_suffix='b')))

    def parse_int_array(self):
        """Parse an int array from the token stream."""
        return IntArray(list(self.array_items('int')))

    def parse_long_array(self):
        """Parse a long array from the token stream."""
        return LongArray(list(self.array_items('long', number_suffix='l')))

    def parse_list(self):
        """Parse a list from the token stream."""
        list_tag = List[End]()

        for token in self.collect_tokens_until('CLOSE_BRACKET'):
            item = self.parse()
            if list_tag.subtype is End:
                list_tag = List[item.__class__]()
            elif not isinstance(item, list_tag.subtype):
                raise self.error(f'Item {str(item)!r} is not a '
                                 f'{list_tag.subtype.__name__} tag')
            list_tag.append(item)
        return list_tag

    def parse_invalid(self):
        """Parse an invalid token from the token stream."""
        raise self.error(f'Invalid token {self.current_token.value!r}')

    @staticmethod
    def unquote_string(string):
        """Return the unquoted value of a quoted string."""
        return json.loads(string.replace(r'\\', '\\'))  # TODO: Fix escaping
<commit_msg>Allow omitting the suffix for doubles<commit_after>
"""This module exposes utilities for parsing literal nbt strings.

Exported items:
    parse_nbt      -- Helper function that parses nbt tags
    InvalidLiteral -- Exception raised when parsing invalid nbt literals
"""

__all__ = ['parse_nbt', 'InvalidLiteral']

import re
import json
from collections import namedtuple

from .tag import *

# Token definition

TOKENS = {
    'QUOTED_STRING': r'""|".*?[^\\]"',
    'NUMBER': r'[+-]?(?:[0-9]*?\.[0-9]+|[0-9]+\.[0-9]*?|[0-9]+)[bslfdBSLFD]?',
    'STRING': r'[a-zA-Z0-9._+-]+',
    'COMPOUND': r'\{',
    'CLOSE_COMPOUND': r'\}',
    'BYTE_ARRAY': r'\[B;',
    'INT_ARRAY': r'\[I;',
    'LONG_ARRAY': r'\[L;',
    'LIST': r'\[',
    'CLOSE_BRACKET': r'\]',
    'COLON': r':',
    'COMMA': r',',
    'INVALID': r'.+?',
}

# Build the regex

TOKENS_REGEX = re.compile(
    '|'.join(f'\s*(?P<{key}>{value})\s*' for key, value in TOKENS.items())
)

# Associate number suffixes to tag types

NUMBER_SUFFIXES = {'b': Byte, 's': Short, 'l': Long, 'f': Float, 'd': Double}

def parse_nbt(literal):
    """Parse a literal nbt string and return the resulting tag."""
    parser = NbtParser(tokenize(literal))
    tag = parser.parse()

    cursor = parser.token_span[1]
    leftover = literal[cursor:]
    if leftover.strip():
        parser.token_span = cursor, cursor + len(leftover)
        raise parser.error(f'Expected end of string but got {leftover!r}')

    return tag

class InvalidLiteral(ValueError):
    """Exception raised when parsing invalid nbt literals.

    The exception must be instantiated with two parameters. The first
    one needs to be a tuple representing the location of the error in
    the nbt string (start_index, end_index). The second argument is the
    actual error message.
    """

    def __str__(self):
        return f'{self.args[1]} at position {self.args[0][0]}'

Token = namedtuple('Token', ['type', 'value', 'span'])

def tokenize(string):
    """Match and yield all the tokens of the input string."""
    for match in TOKENS_REGEX.finditer(string):
        yield Token(match.lastgroup, match.group().strip(), match.span())

class NbtParser:
    """Nbt literal parser.

    The parser needs to be instantiated with a token stream as argument.
    Using the `parse` method will return the corresponding nbt tag.

    The parser will raise an InvalidLiteral exception if it encounters
    an invalid nbt literal while parsing.
    """

    def __init__(self, token_stream):
        self.token_stream = iter(token_stream)
        self.current_token = None
        self.token_span = (0, 0)

        self.next()

    def error(self, message):
        """Create an InvalidLiteral using the current token position."""
        return InvalidLiteral(self.token_span, message)

    def next(self):
        """Move to the next token in the token stream."""
        self.current_token = next(self.token_stream, None)
        if self.current_token is None:
            self.token_span = self.token_span[1], self.token_span[1]
            raise self.error('Unexpected end of input')
        self.token_span = self.current_token.span
        return self

    def parse(self):
        """Parse and return an nbt literal from the token stream."""
        token_type = self.current_token.type.lower()
        handler = getattr(self, f'parse_{token_type}', None)
        if handler is None:
            raise self.error(f'Invalid literal {self.current_token.value!r}')
        return handler()

    def parse_quoted_string(self):
        """Parse a quoted string from the token stream."""
        return String(self.unquote_string(self.current_token.value))

    def parse_number(self):
        """Parse a number from the token stream."""
        value = self.current_token.value
        suffix = value[-1].lower()
        if suffix in NUMBER_SUFFIXES:
            return NUMBER_SUFFIXES[suffix](value[:-1])
        else:
            return Double(value) if '.' in value else Int(value)

    def parse_string(self):
        """Parse a regular unquoted string from the token stream."""
        return String(self.current_token.value)

    def collect_tokens_until(self, token_type):
        """Yield the item tokens in a comma-separated tag collection."""
        self.next()
        if self.current_token.type == token_type:
            return

        while True:
            yield self.current_token

            self.next()
            if self.current_token.type == token_type:
                return

            if self.current_token.type != 'COMMA':
                raise self.error(f'Expected comma but got '
                                 f'{self.current_token.value!r}')
            self.next()

    def parse_compound(self):
        """Parse a compound from the token stream."""
        compound_tag = Compound()

        for token in self.collect_tokens_until('CLOSE_COMPOUND'):
            item_key = token.value
            if token.type not in ('STRING', 'QUOTED_STRING'):
                raise self.error(f'Expected compound key but got {item_key!r}')

            if token.type == 'QUOTED_STRING':
                item_key = self.unquote_string(item_key)

            if self.next().current_token.type != 'COLON':
                raise self.error(f'Expected colon but got '
                                 f'{self.current_token.value!r}')
            self.next()
            compound_tag[item_key] = self.parse()
        return compound_tag

    def array_items(self, number_type, *, number_suffix=''):
        """Parse and yield array items from the token stream."""
        for token in self.collect_tokens_until('CLOSE_BRACKET'):
            is_number = token.type == 'NUMBER'
            value = token.value.lower()
            if not (is_number and value.endswith(number_suffix)):
                raise self.error(f'Invalid {number_type} array element '
                                 f'{token.value!r}')
            yield int(value.replace(number_suffix, ''))

    def parse_byte_array(self):
        """Parse a byte array from the token stream."""
        return ByteArray(list(self.array_items('byte', number_suffix='b')))

    def parse_int_array(self):
        """Parse an int array from the token stream."""
        return IntArray(list(self.array_items('int')))

    def parse_long_array(self):
        """Parse a long array from the token stream."""
        return LongArray(list(self.array_items('long', number_suffix='l')))

    def parse_list(self):
        """Parse a list from the token stream."""
        list_tag = List[End]()

        for token in self.collect_tokens_until('CLOSE_BRACKET'):
            item = self.parse()
            if list_tag.subtype is End:
                list_tag = List[item.__class__]()
            elif not isinstance(item, list_tag.subtype):
                raise self.error(f'Item {str(item)!r} is not a '
                                 f'{list_tag.subtype.__name__} tag')
            list_tag.append(item)
        return list_tag

    def parse_invalid(self):
        """Parse an invalid token from the token stream."""
        raise self.error(f'Invalid token {self.current_token.value!r}')

    @staticmethod
    def unquote_string(string):
        """Return the unquoted value of a quoted string."""
        return json.loads(string.replace(r'\\', '\\'))  # TODO: Fix escaping
<|endoftext|>


<commit_before>#!/usr/bin/env python
#
# igcollect - Linux network
#
# Copyright (c) 2018, InnoGames GmbH
#

from argparse import ArgumentParser
from time import time
from os import listdir
from os.path import isdir, islink, join

class InterfaceStatistics(object):
    def _check_dir(self, dev, directory):
        return isdir(join(self._scn, dev, directory))

    def _check_name(self, dev, name):
        return dev == name

    def _check_symlink(self, dev, symlink):
        return islink(join(self._scn, dev, symlink))

    def _check_type(self, dev, types):
        with open(join(self._scn, dev, 'type'), 'r') as fd:
            dev_type = fd.readline().strip()
        return any(dev_type == t for t in types)

    def _check_uevent(self, dev, string):
        with open(join(self._scn, dev, 'uevent'), 'r') as fd:
            return string in fd.read()

    def _read_stat(self, dev, param):
        with open(join(self._scn, dev, 'statistics', param)) as fd:
            return int(fd.readline().strip())

    # Supported types of interfaces for a metrics sending
    # If there is more than one conditions they multipied by AND
    NET_TYPES = {
        'bond': (_check_dir, 'bonding'),
        'bond_slave': (_check_dir, 'bonding_slave'),
        'bridge': (_check_dir, 'bridge'),
        'bridge_slave': (_check_dir, 'brport'),
        'general_slave': (_check_symlink, 'master'),
        'lo': (_check_type, ('772', )),
        'ovs-br0': (_check_name, 'br0'),
        'ovs-system': (_check_name, 'ovs-system'),
        'phys': (_check_symlink, 'device'),
        'tunnel': (_check_type, ('768', '776')),
        'vlan': (_check_uevent, "DEVTYPE=vlan"),
    }
    _scn = '/sys/class/net'

    def __init__(self, included_types=[]):
        self.included_types = included_types
        self.netdev_stat = {}

    def get_interfaces(self):
        for dev in listdir(self._scn):
            if self.included_types:
                for i_type in self.included_types:
                    checks = self.NET_TYPES[i_type]
                    results = []
                    for check, arg in zip(checks[::2], checks[1::2]):
                        results.append(check(self, dev, arg))
                    if False not in results:
                        self.netdev_stat[dev] = {}
            else:
                self.netdev_stat[dev] = {}

    def fill_metrics(self):
        self.timestamp = int(time())
        metric_names = {
            'bytesIn': ['rx_bytes'],
            'bytesOut': ['tx_bytes'],
            'pktsIn': ['rx_packets'],
            'pktsOut': ['tx_packets'],
            'errsIn': ['rx_errors'],
            'errsOut': ['tx_errors'],
            'dropIn': [
                'rx_dropped',
                'rx_missed_errors'
            ],
            'dropOut': ['tx_dropped'],
            'fifoIn': ['rx_fifo_errors'],
            'fifoOut': ['tx_fifo_errors'],
            'frameIn': [
                'rx_length_errors',
                'rx_over_errors',
                'rx_crc_errors',
                'rx_frame_errors'
            ],
            'collsOut': ['collisions'],
            'carrierOut': [
                'tx_carrier_errors',
                'tx_aborted_errors',
                'tx_window_errors',
                'tx_heartbeat_errors'
            ],
        }
        virt_metrics = [
            'bytesIn',
            'bytesOut',
            'pktsIn',
            'pktsOut',
        ]
        if self.netdev_stat == {}:
            self.get_interfaces()
        for dev in self.netdev_stat:
            # We send all metrics only for physical devices
            if self._check_symlink(dev, 'device'):
                dev_metrics = metric_names.keys()
            else:
                dev_metrics = virt_metrics
            for m in dev_metrics:
                self.netdev_stat[dev][m] = 0
                for param in metric_names[m]:
                    self.netdev_stat[dev][m] += self._read_stat(dev, param)

    def print_metrics(self, prefix):
        for dev in self.netdev_stat:
            for metric in self.netdev_stat[dev]:
                print('{}.{}.{} {} {}'.format(
                    prefix, dev.replace('.', '_'), metric,
                    self.netdev_stat[dev][metric], self.timestamp
                ))

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('--prefix', default='network')
    parser.add_argument('--enabled-types', action='append',
                        default=[],
                        choices=InterfaceStatistics.NET_TYPES.keys(),
                        help='list of enabled interfaces')
    return parser.parse_args()

def main():
    args = parse_args()
    ns = InterfaceStatistics(args.enabled_types)
    ns.fill_metrics()
    ns.print_metrics(args.prefix)

if __name__ == '__main__':
    main()
<commit_msg>Fix skipping for general files in /sys/class/net * https://www.kernel.org/doc/Documentation/networking/bonding.txt - /sys/class/net/bonding_masters<commit_after>#!/usr/bin/env python
#
# igcollect - Linux network
#
# Copyright (c) 2018, InnoGames GmbH
#

from argparse import ArgumentParser
from time import time
from os import listdir
from os.path import isdir, islink, join

class InterfaceStatistics(object):
    def _check_dir(self, dev, directory):
        return isdir(join(self._scn, dev, directory))

    def _check_name(self, dev, name):
        return dev == name

    def _check_symlink(self, dev, symlink):
        return islink(join(self._scn, dev, symlink))

    def _check_type(self, dev, types):
        with open(join(self._scn, dev, 'type'), 'r') as fd:
            dev_type = fd.readline().strip()
        return any(dev_type == t for t in types)

    def _check_uevent(self, dev, string):
        with open(join(self._scn, dev, 'uevent'), 'r') as fd:
            return string in fd.read()

    def _read_stat(self, dev, param):
        with open(join(self._scn, dev, 'statistics', param)) as fd:
            return int(fd.readline().strip())

    # Supported types of interfaces for a metrics sending
    # If there is more than one conditions they multipied by AND
    NET_TYPES = {
        'bond': (_check_dir, 'bonding'),
        'bond_slave': (_check_dir, 'bonding_slave'),
        'bridge': (_check_dir, 'bridge'),
        'bridge_slave': (_check_dir, 'brport'),
        'general_slave': (_check_symlink, 'master'),
        'lo': (_check_type, ('772', )),
        'ovs-br0': (_check_name, 'br0'),
        'ovs-system': (_check_name, 'ovs-system'),
        'phys': (_check_symlink, 'device'),
        'tunnel': (_check_type, ('768', '776')),
        'vlan': (_check_uevent, "DEVTYPE=vlan"),
    }
    _scn = '/sys/class/net'

    def __init__(self, included_types=[]):
        self.included_types = included_types
        self.netdev_stat = {}

    def get_interfaces(self):
        for dev in listdir(self._scn):
            dev_path = join(self._scn, dev)
            if not (islink(dev_path) or isdir(dev_path)):
                # Skip regular files
                continue
            if not self.included_types:
                # Send metrics for all devices
                self.netdev_stat[dev] = {}
                continue
            for i_type in self.included_types:
                checks = self.NET_TYPES[i_type]
                results = []
                for check, arg in zip(checks[::2], checks[1::2]):
                    results.append(check(self, dev, arg))
                if False not in results:
                    self.netdev_stat[dev] = {}

    def fill_metrics(self):
        self.timestamp = int(time())
        metric_names = {
            'bytesIn': ['rx_bytes'],
            'bytesOut': ['tx_bytes'],
            'pktsIn': ['rx_packets'],
            'pktsOut': ['tx_packets'],
            'errsIn': ['rx_errors'],
            'errsOut': ['tx_errors'],
            'dropIn': [
                'rx_dropped',
                'rx_missed_errors'
            ],
            'dropOut': ['tx_dropped'],
            'fifoIn': ['rx_fifo_errors'],
            'fifoOut': ['tx_fifo_errors'],
            'frameIn': [
                'rx_length_errors',
                'rx_over_errors',
                'rx_crc_errors',
                'rx_frame_errors'
            ],
            'collsOut': ['collisions'],
            'carrierOut': [
                'tx_carrier_errors',
                'tx_aborted_errors',
                'tx_window_errors',
                'tx_heartbeat_errors'
            ],
        }
        virt_metrics = [
            'bytesIn',
            'bytesOut',
            'pktsIn',
            'pktsOut',
        ]
        if self.netdev_stat == {}:
            self.get_interfaces()
        for dev in self.netdev_stat:
            # We send all metrics only for physical devices
            if self._check_symlink(dev, 'device'):
                dev_metrics = metric_names.keys()
            else:
                dev_metrics = virt_metrics
            for m in dev_metrics:
                self.netdev_stat[dev][m] = 0
                for param in metric_names[m]:
                    self.netdev_stat[dev][m] += self._read_stat(dev, param)

    def print_metrics(self, prefix):
        for dev in self.netdev_stat:
            for metric in self.netdev_stat[dev]:
                print('{}.{}.{} {} {}'.format(
                    prefix, dev.replace('.', '_'), metric,
                    self.netdev_stat[dev][metric], self.timestamp
                ))

def parse_args():
    parser = ArgumentParser()
    parser.add_argument('--prefix', default='network')
    parser.add_argument('--enabled-types', action='append',
                        default=[],
                        choices=InterfaceStatistics.NET_TYPES.keys(),
                        help='list of enabled interfaces')
    return parser.parse_args()

def main():
    args = parse_args()
    ns = InterfaceStatistics(args.enabled_types)
    ns.fill_metrics()
    ns.print_metrics(args.prefix)

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
"""
To use this script you'll need flac and lame
Usage :
flac2mp3 [origin directories containing flac files] mp3/repository/destination
"""
import getopt
from itertools import repeat
from logging import INFO
import logging
from multiprocessing import Pool
import multiprocessing
import re
from struct import unpack, unpack_from
import sys
from tempfile import mkstemp
from genericpath import isdir, isfile
import os
from os.path import dirname, join
from subprocess import Popen, PIPE

__author__ = 'bruno thomas'

logging.basicConfig(format='%(asctime)s [%(name)s] %(levelname)s: %(message)s')
LOGGER = logging.getLogger('flac2mp3')

LAME_COMMAND = 'lame --silent -V2 --vbr-new -q0 --lowpass 19.7 --resample 44100 --add-id3v2'
VOBIS_COMMENT = 4
PICTURE = 6

class none_if_missing(dict):
    def __missing__(self, _):return None

vobis_comments_lame_opts_map = none_if_missing({
   'ARTIST'     : '--ta',
   'ALBUM'      : '--tl',
   'TITLE'      : '--tt',
   'DESCRIPTION': '--tc',
   'GENRE'      : '--tg',
   'DATE'       : '--ty',
   'TRACKNUMBER': '--tn',
   'TRACKTOTAL' : 'total'
})

class VobisCommentParser(object):
    image = None
    flac_tags = {}
    def parse(self, flac_file):
        with open(flac_file, 'rb') as flac:
            assert 'fLaC' == flac.read(4)

            last_block = False
            block_type = 0

            while not last_block:
                last_block_and_block_type = flac.read(1)
                block_type = ord(last_block_and_block_type) & 0x07
                last_block = ord(last_block_and_block_type) & 0x80 is 0x80
                block_length, = unpack('>i', '\x00' + flac.read(3))
                block = flac.read(int(block_length))
                if block_type is VOBIS_COMMENT:
                    self.flac_tags = self.get_flac_tags(block)
                if block_type is PICTURE:
                    self.image = self.get_image_data(block)

        if not self.flac_tags:
            raise RuntimeError('cannot find vobis comment vobis_comment_block in %s' % flac_file)
        return self

    def get_image_data(self, image_block):
        offset = 4
        mime_type_length, = unpack_from('>i', image_block, offset)
        offset += 4 + mime_type_length
        description_length, = unpack_from('>i', image_block, offset)
        offset += 20 + description_length
        image_lenth, = unpack_from('>i', image_block, offset)
        offset += 4
        return image_block[offset:offset + image_lenth]

    def get_flac_tags(self, vobis_comment_block):
        vendor_length, = unpack('I', vobis_comment_block[0:4])
        offset = 4 + vendor_length
        user_comment_list_length, = unpack('I', vobis_comment_block[offset:offset + 4])
        offset += 4
        comments = list()
        for vobis_comment_index in range(user_comment_list_length):
            length, = unpack('I', vobis_comment_block[offset:offset + 4])
            offset += 4
            comments.append(vobis_comment_block[offset:offset + length])
            offset += length
        return none_if_missing(split_key_value_at_first_equal_and_upper_key(comment) for comment in comments)

class CoverFile(object):
    cover_file = None
    image_data = None
    tmp_prefix = 'flac2mp3'
    tmp_suffix = '.tmp'

    def __init__(self, flac_file, image_data):
        cover_file = join(dirname(flac_file), "cover.jpg")
        if os.path.isfile(cover_file):
            self.cover_file = cover_file
        elif image_data:
            self.image_data = image_data
            _,self.cover_file = mkstemp(prefix=self.tmp_prefix, suffix=self.tmp_suffix)

    def __enter__(self):
        if self.image_data:
            with open(self.cover_file, 'wb') as cover:
                cover.write(self.image_data)
        return self

    def __exit__(self, type, value, traceback):
        if self.image_data: os.remove(self.cover_file)

    def exist(self): return self.cover_file is not None
    def path(self): return self.cover_file

def transcode(flac_file, mp3_file):
    parser = VobisCommentParser().parse(flac_file)
    LOGGER.info('transcoding %s with tags (title=%s artist=%s track=%s/%s)', flac_file, parser.flac_tags['TITLE'], parser.flac_tags['ARTIST'], parser.flac_tags['TRACKNUMBER'], parser.flac_tags['TRACKTOTAL'])

    lame_tags = {vobis_comments_lame_opts_map[k]: v for k,v in parser.flac_tags.items()}
    if 'total' in lame_tags:
        lame_tags['--tn'] = '%s/%s' % (parser.flac_tags['TRACKNUMBER'], lame_tags.pop('total'))

    with CoverFile(flac_file, parser.image) as cover_file:
        if cover_file.exist(): lame_tags['--ti'] = cover_file.path()

        lame_command_list = LAME_COMMAND.split(' ')
        lame_command_list.extend(arg for k,v in lame_tags.items() if k for arg in (k,v))
        lame_command_list.extend(('-', mp3_file))

        flac_command = Popen(('flac', '--totally-silent', '-dc', flac_file), stdout=PIPE)
        lame_command = Popen(lame_command_list, stdin=flac_command.stdout)
        lame_command.wait()

def find_files(pattern, *root_dirs):
    regexp = re.compile(pattern)
    for root_dir in root_dirs:
        for root, _, files in os.walk(root_dir):
            for file in files:
                if regexp.match(file): yield join(root, file)

def get_mp3_filename(mp3_target_path, flac_root_path, flac_file):
    flac_path_relative_to_root = flac_file.replace(flac_root_path, '').replace('.flac', '.mp3')
    if flac_path_relative_to_root.startswith('/'): flac_path_relative_to_root = flac_path_relative_to_root[1:]
    return join(mp3_target_path, flac_path_relative_to_root)

def tags_are_equals(flac_file, target_mp3_file):
    try:
        import eyeD3
        mp3_tags = eyeD3.Tag()
        mp3_tags.link(target_mp3_file)
        parser = VobisCommentParser().parse(flac_file)

        return \
            parser.flac_tags['ARTIST'] == mp3_tags.getArtist() and \
            parser.flac_tags['ALBUM'] == mp3_tags.getAlbum() and \
            parser.flac_tags['TITLE'] == mp3_tags.getTitle() and\
            parser.flac_tags['GENRE'] == mp3_tags.getGenre().name and \
            parser.flac_tags['DATE'] == mp3_tags.getYear() and \
            int(parser.flac_tags['TRACKNUMBER']) == mp3_tags.getTrackNum()[0] and \
            (not parser.flac_tags['TRACKTOTAL'] or int(parser.flac_tags['TRACKTOTAL']) == mp3_tags.getTrackNum()[1])
    except ImportError:
        return False

def process_transcoding((flac_file, flac_root_path, mp3_target_path)):
    try:
        target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file)
        if not isdir(dirname(target_mp3_file)):
            try:
                os.makedirs(dirname(target_mp3_file))
            except OSError:
                pass # other thread might have been faster
        if isfile(target_mp3_file) and tags_are_equals(flac_file, target_mp3_file):
            LOGGER.info('skipping %r as target mp3 file exists and seems to have the same tags', flac_file)
        else:
            transcode(flac_file, target_mp3_file)
    except Exception as e:
        LOGGER.exception('error during the transcoding of %r : %s' % (flac_file, e))

def which(program):
    def is_exe(fpath): return isfile(fpath) and os.access(fpath, os.X_OK)

    for path in os.environ["PATH"].split(os.pathsep):
        exe_files = os.path.join(path, program), os.path.join(path, program + '.exe')
        for exe_file in exe_files:
            if is_exe(exe_file):
                return exe_file

def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    except NotImplementedError:
        return 1

def run(mp3_target_path, flac_root_path, *flac_path_list):
    flac_files = set(find_files('.*\.flac', *flac_path_list))
    cpu_count = get_cpu_count()
    LOGGER.info('found %d cpu(s)', cpu_count)
    LOGGER.info('found %d flac files', len(flac_files))
    LOGGER.info('transcoding files with command "%s"', LAME_COMMAND)

    Pool(cpu_count).map(process_transcoding, zip(flac_files, repeat(flac_root_path), repeat(mp3_target_path)))

    LOGGER.info('transcoding done, exiting normally')

def split_key_value_at_first_equal_and_upper_key(string_with_equal):
    k,v = string_with_equal.split('=', 1)
    # vobis comments are utf-8 http://www.xiph.org/vorbis/doc/v-comment.html
    if k.upper() in ('ARTIST', 'ALBUM', 'TITLE'): v = v.decode('utf-8')
    return k.upper(), v

class Usage(Exception):
    def __init__(self, msg, *args, **kwargs):
        super(Usage, self).__init__(*args, **kwargs)
        self.msg = msg

def main(argv):
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
        except getopt.error, msg:
            raise Usage(msg)

        for opt, arg in opts:
            if opt in ("-h", "--help"):
                print __doc__
                return 0

        if args:
            mp3_target_path = args.pop()
        else:
            raise Usage('no mp3 target directory given')

        if not which('lame'):
            LOGGER.fatal("Cannot find lame. Please install lame: http://lame.sourceforge.net/")
            return 3
        if not which('flac'):
            LOGGER.fatal("Cannot find flac. Please install flac: http://flac.sourceforge.net/")
            return 3

        run(mp3_target_path, os.getcwd(), *args)
        return 0
    except Usage, err:
        print >> sys.stderr, err.msg
        print >> sys.stderr, "for help use -h or --help"
        return 2

if __name__ == "__main__":
    LOGGER.setLevel(INFO)
    sys.exit(main(sys.argv))
<commit_msg>[encoding] display file with filesystem encoding<commit_after>#!/usr/bin/env python
"""
To use this script you'll need flac and lame
Usage :
flac2mp3 [origin directories containing flac files] mp3/repository/destination
"""
import getopt
from itertools import repeat
from logging import INFO
import logging
from multiprocessing import Pool
import multiprocessing
import locale
import re
from struct import unpack, unpack_from
import sys
from tempfile import mkstemp
from genericpath import isdir, isfile
import os
from os.path import dirname, join
from subprocess import Popen, PIPE

__author__ = 'bruno thomas'

logging.basicConfig(format='%(asctime)s [%(name)s] %(levelname)s: %(message)s')
LOGGER = logging.getLogger('flac2mp3')

LAME_COMMAND = 'lame --silent -V2 --vbr-new -q0 --lowpass 19.7 --resample 44100 --add-id3v2'
VOBIS_COMMENT = 4
PICTURE = 6

class none_if_missing(dict):
    def __missing__(self, _):return None

vobis_comments_lame_opts_map = none_if_missing({
   'ARTIST'     : '--ta',
   'ALBUM'      : '--tl',
   'TITLE'      : '--tt',
   'DESCRIPTION': '--tc',
   'GENRE'      : '--tg',
   'DATE'       : '--ty',
   'TRACKNUMBER': '--tn',
   'TRACKTOTAL' : 'total'
})

class VobisCommentParser(object):
    image = None
    flac_tags = {}
    def parse(self, flac_file):
        with open(flac_file, 'rb') as flac:
            assert 'fLaC' == flac.read(4)

            last_block = False
            block_type = 0

            while not last_block:
                last_block_and_block_type = flac.read(1)
                block_type = ord(last_block_and_block_type) & 0x07
                last_block = ord(last_block_and_block_type) & 0x80 is 0x80
                block_length, = unpack('>i', '\x00' + flac.read(3))
                block = flac.read(int(block_length))
                if block_type is VOBIS_COMMENT:
                    self.flac_tags = self.get_flac_tags(block)
                if block_type is PICTURE:
                    self.image = self.get_image_data(block)

        if not self.flac_tags:
            raise RuntimeError('cannot find vobis comment vobis_comment_block in %s' % flac_file)
        return self

    def get_image_data(self, image_block):
        offset = 4
        mime_type_length, = unpack_from('>i', image_block, offset)
        offset += 4 + mime_type_length
        description_length, = unpack_from('>i', image_block, offset)
        offset += 20 + description_length
        image_lenth, = unpack_from('>i', image_block, offset)
        offset += 4
        return image_block[offset:offset + image_lenth]

    def get_flac_tags(self, vobis_comment_block):
        vendor_length, = unpack('I', vobis_comment_block[0:4])
        offset = 4 + vendor_length
        user_comment_list_length, = unpack('I', vobis_comment_block[offset:offset + 4])
        offset += 4
        comments = list()
        for vobis_comment_index in range(user_comment_list_length):
            length, = unpack('I', vobis_comment_block[offset:offset + 4])
            offset += 4
            comments.append(vobis_comment_block[offset:offset + length])
            offset += length
        return none_if_missing(split_key_value_at_first_equal_and_upper_key(comment) for comment in comments)

class CoverFile(object):
    cover_file = None
    image_data = None
    tmp_prefix = 'flac2mp3'
    tmp_suffix = '.tmp'

    def __init__(self, flac_file, image_data):
        cover_file = join(dirname(flac_file), "cover.jpg")
        if os.path.isfile(cover_file):
            self.cover_file = cover_file
        elif image_data:
            self.image_data = image_data
            _,self.cover_file = mkstemp(prefix=self.tmp_prefix, suffix=self.tmp_suffix)

    def __enter__(self):
        if self.image_data:
            with open(self.cover_file, 'wb') as cover:
                cover.write(self.image_data)
        return self

    def __exit__(self, type, value, traceback):
        if self.image_data: os.remove(self.cover_file)

    def exist(self): return self.cover_file is not None
    def path(self): return self.cover_file

def transcode(flac_file, mp3_file):
    parser = VobisCommentParser().parse(flac_file)
    fs_encoding = sys.getfilesystemencoding()
    LOGGER.info('transcoding "%s" with tags (title="%s" artist="%s" track=%s/%s)', flac_file.decode(fs_encoding), parser.flac_tags['TITLE'], parser.flac_tags['ARTIST'], parser.flac_tags['TRACKNUMBER'], parser.flac_tags['TRACKTOTAL'])

    lame_tags = {vobis_comments_lame_opts_map[k]: v for k,v in parser.flac_tags.items()}
    if 'total' in lame_tags:
        lame_tags['--tn'] = '%s/%s' % (parser.flac_tags['TRACKNUMBER'], lame_tags.pop('total'))

    with CoverFile(flac_file, parser.image) as cover_file:
        if cover_file.exist(): lame_tags['--ti'] = cover_file.path()

        lame_command_list = LAME_COMMAND.split(' ')
        lame_command_list.extend(arg for k,v in lame_tags.items() if k for arg in (k,v))
        lame_command_list.extend(('-', mp3_file))

        flac_command = Popen(('flac', '--totally-silent', '-dc', flac_file), stdout=PIPE)
        lame_command = Popen(lame_command_list, stdin=flac_command.stdout)
        lame_command.wait()

def find_files(pattern, *root_dirs):
    regexp = re.compile(pattern)
    for root_dir in root_dirs:
        for root, _, files in os.walk(root_dir):
            for file in files:
                if regexp.match(file): yield join(root, file)

def get_mp3_filename(mp3_target_path, flac_root_path, flac_file):
    flac_path_relative_to_root = flac_file.replace(flac_root_path, '').replace('.flac', '.mp3')
    if flac_path_relative_to_root.startswith('/'): flac_path_relative_to_root = flac_path_relative_to_root[1:]
    return join(mp3_target_path, flac_path_relative_to_root)

def tags_are_equals(flac_file, target_mp3_file):
    try:
        import eyeD3
        mp3_tags = eyeD3.Tag()
        mp3_tags.link(target_mp3_file)
        parser = VobisCommentParser().parse(flac_file)

        return \
            parser.flac_tags['ARTIST'] == mp3_tags.getArtist() and \
            parser.flac_tags['ALBUM'] == mp3_tags.getAlbum() and \
            parser.flac_tags['TITLE'] == mp3_tags.getTitle() and\
            parser.flac_tags['GENRE'] == mp3_tags.getGenre().name and \
            parser.flac_tags['DATE'] == mp3_tags.getYear() and \
            int(parser.flac_tags['TRACKNUMBER']) == mp3_tags.getTrackNum()[0] and \
            (not parser.flac_tags['TRACKTOTAL'] or int(parser.flac_tags['TRACKTOTAL']) == mp3_tags.getTrackNum()[1])
    except ImportError:
        return False

def process_transcoding((flac_file, flac_root_path, mp3_target_path)):
    try:
        target_mp3_file = get_mp3_filename(mp3_target_path, flac_root_path, flac_file)
        if not isdir(dirname(target_mp3_file)):
            try:
                os.makedirs(dirname(target_mp3_file))
            except OSError:
                pass # other thread might have been faster
        if isfile(target_mp3_file) and tags_are_equals(flac_file, target_mp3_file):
            LOGGER.info('skipping %r as target mp3 file exists and seems to have the same tags', flac_file)
        else:
            transcode(flac_file, target_mp3_file)
    except Exception as e:
        LOGGER.exception('error during the transcoding of %r : %s' % (flac_file, e))

def which(program):
    def is_exe(fpath): return isfile(fpath) and os.access(fpath, os.X_OK)

    for path in os.environ["PATH"].split(os.pathsep):
        exe_files = os.path.join(path, program), os.path.join(path, program + '.exe')
        for exe_file in exe_files:
            if is_exe(exe_file):
                return exe_file

def get_cpu_count():
    try:
        return multiprocessing.cpu_count()
    except NotImplementedError:
        return 1

def run(mp3_target_path, flac_root_path, *flac_path_list):
    flac_files = set(find_files('.*\.flac', *flac_path_list))
    cpu_count = get_cpu_count()
    LOGGER.info('found %d cpu(s)', cpu_count)
    LOGGER.info('found %d flac files', len(flac_files))
    LOGGER.info('transcoding files with command "%s"', LAME_COMMAND)

    Pool(cpu_count).map(process_transcoding, zip(flac_files, repeat(flac_root_path), repeat(mp3_target_path)))

    LOGGER.info('transcoding done, exiting normally')

def split_key_value_at_first_equal_and_upper_key(string_with_equal):
    k,v = string_with_equal.split('=', 1)
    # vobis comments are utf-8 http://www.xiph.org/vorbis/doc/v-comment.html
    if k.upper() in ('ARTIST', 'ALBUM', 'TITLE'): v = v.decode('utf-8')
    return k.upper(), v

class Usage(Exception):
    def __init__(self, msg, *args, **kwargs):
        super(Usage, self).__init__(*args, **kwargs)
        self.msg = msg

def main(argv):
    try:
        try:
            opts, args = getopt.getopt(argv[1:], "h", ["help"])
        except getopt.error, msg:
            raise Usage(msg)

        for opt, arg in opts:
            if opt in ("-h", "--help"):
                print __doc__
                return 0

        if args:
            mp3_target_path = args.pop()
        else:
            raise Usage('no mp3 target directory given')

        if not which('lame'):
            LOGGER.fatal("Cannot find lame. Please install lame: http://lame.sourceforge.net/")
            return 3
        if not which('flac'):
            LOGGER.fatal("Cannot find flac. Please install flac: http://flac.sourceforge.net/")
            return 3

        run(mp3_target_path, os.getcwd(), *args)
        return 0
    except Usage, err:
        print >> sys.stderr, err.msg
        print >> sys.stderr, "for help use -h or --help"
        return 2

if __name__ == "__main__":
    LOGGER.setLevel(INFO)
    sys.exit(main(sys.argv))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 Simone Marsili
# All rights reserved.
# License: BSD 3 clause
"""Base classes module."""
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)
import numpy
from ndd.base import BaseEstimator
import ndd.fnsb

class EntropyEstimatorMixin(object):
    @staticmethod
    def _plugin_estimator(pk, k):
        return ndd.fnsb.plugin(pk, k), None

    @staticmethod
    def _pseudocounts_estimator(pk, k, alpha):
        return ndd.fnsb.pseudo(pk, k, alpha), None

    @staticmethod
    def _ww_estimator(pk, k, alpha):
        return ndd.fnsb.dirichlet(pk, k, alpha), None

    @staticmethod
    def _nsb_estimator(pk, k):
        return ndd.fnsb.nsb(pk, k)

    def estimator(self, pk, k):
        if self._estimator is None:
            if self.plugin:
                if self.alpha is None:
                    self._estimator = self._plugin_estimator
                else:
                    self._estimator = lambda pk, k: self._pseudocounts_estimator(pk, k, self.alpha)  # pylint: disable=redefined-variable-type
            else:
                if self.alpha is None:
                    self._estimator = self._nsb_estimator
                else:
                    self._estimator = lambda pk, k: self._ww_estimator(pk, k, self.alpha)
        return self._estimator(pk, k)

    @staticmethod
    def _check_k(k, n_bins):
        """
        if k is None, set k = number of bins
        if k is an integer, just check
        ik an array set k = prod(k)
        """
        MAX_LOGK = 150 * numpy.log(2)

        if k is None:
            # set k to the number of observed bins
            k = numpy.float64(n_bins)
        else:
            try:
                k = numpy.float64(k)
            except ValueError:
                raise
            if k.ndim:
                # if k is a sequence, set k = prod(k)
                if k.ndim > 1:
                    raise ValueError('k must be a scalar or 1D array')
                logk = numpy.sum(numpy.log(x) for x in k)
                if logk > MAX_LOGK:
                    # too large a number; backoff to n_bins?
                    # TODO: log warning
                    raise ValueError('k (%r) larger than %r' %
                                     (numpy.exp(logk), numpy.exp(MAX_LOGK)))
                else:
                    k = numpy.prod(k)
            else:
                # if a scalar check size
                if numpy.log(k) > MAX_LOGK:
                    raise ValueError('k (%r) larger than %r' %
                                     (k, numpy.exp(MAX_LOGK)))
            # consistency checks
            if k < n_bins:
                raise ValueError("k (%s) is smaller than the number of bins"
                                 "(%s)" % (k, n_bins))
            if not k.is_integer():
                raise ValueError("k (%s) should be a whole number." % k)
        return k

# TODO: docstrings
class Entropy(EntropyEstimatorMixin, BaseEstimator):
    def __init__(self, alpha=None, plugin=False):

        self.estimate = None
        self.std = None
        self._estimator = None

        # check alpha value
        if alpha:
            try:
                alpha = numpy.float64(alpha)
            except ValueError:
                raise
            if alpha <= 0:
                raise ValueError("alpha <= 0")
        self.alpha = alpha
        self.plugin = plugin

    def _check_input(self, pk, k):
        pk = self._check_counts(a=pk)
        k = self._check_k(k=k, n_bins=len(pk))
        return pk, k

    @staticmethod
    def _check_counts(a):
        a = numpy.float64(a).flatten()
        not_integers = not numpy.all([x.is_integer() for x in a])
        negative = numpy.any([a < 0])
        if not_integers:
            raise ValueError('counts array has non-integer values')
        if negative:
            raise ValueError('counts array has negative values')
        return numpy.int32(a)

    def fit(self, pk, k=None):
        """
        pk : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(pk).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(pk).

        """
        pk, k = self._check_input(pk, k)
        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            self.estimate, self.std = self.estimator(pk, k)

    def __call__(self, *args, **kwargs):
        """Return estimate from input data. Delegate to fit."""
        self.fit(*args, **kwargs)
        return self.estimate

class KLDivergence(Entropy):
    def __init__(self, qk, alpha=None, plugin=False):
        """qk is a parameter of the estimator; it must be a valid pmf."""
        super().__init__(alpha, plugin)
        if is_pmf(qk):
            self.log_qk = numpy.log(qk)
        else:
            raise ValueError('qk must be a valid PMF')

    def fit(self, pk, k=None):
        """
        pk : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(pk).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(pk).

        """
        pk, k = self._check_input(pk, k)
        if len(self.log_qk) != len(pk):
            raise ValueError('qk and pk must have the same length.')

        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            self.estimate, self.std = self.estimator(pk, k)
        self.estimate -= numpy.sum(pk * self.log_qk)

class JSDivergence(Entropy):
    @staticmethod
    def _check_counts(a):
        a = numpy.float64(a)
        if a.ndim != 2:
            raise ValueError('counts must be 2D.')
        not_integers = not numpy.all([x.is_integer() for x in a.flatten()])
        negative = numpy.any([a < 0])
        if not_integers:
            raise ValueError('counts array has non-integer values')
        if negative:
            raise ValueError('counts array has negative values')
        return numpy.int32(a)

    def _check_input(self, pk, k):
        pk = self._check_counts(a=pk)
        k = self._check_k(k=k, n_bins=pk.shape[1])
        return pk, k

    def fit(self, pk, k=None):
        """
        pk : array_like
            n-by-p array. Different rows correspond to counts from different
            distributions with the same discrete sample space.

        k : int, optional
            Number of bins. k >= p if pk is n-by-p.
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to pk.shape[1].

        """
        pk, k = self._check_input(pk, k)
        ws = numpy.float64(pk.sum(axis=1))
        ws /= ws.sum()
        if k == 1:  # single bin
            self.estimate = 0.0
        else:
            self.estimate = self.estimator(pk.sum(axis=0), k)[0] - sum(
                ws[i]*self.estimator(x, k)[0] for i, x in enumerate(pk))

def _test_JS(n):
    alpha = 0.1
    p = 100
    p1 = numpy.random.dirichlet([alpha]*p)
    p2 = numpy.random.dirichlet([alpha]*p)
    pm = 0.5 * (p1 + p2)

    def ee(x):
        y = - x * numpy.log(x)
        return numpy.sum(y[x > 0])
    js = ee(pm) - 0.5 * (ee(p1) + ee(p2))
    c1 = numpy.random.multinomial(n, p1)
    c2 = numpy.random.multinomial(n, p2)
    pk = numpy.stack([c1, c2])
    est = JSDivergence()
    est.fit(pk)
    js1 = est.estimate
    return js, js1

def is_pmf(a):
    a = numpy.float64(a)
    not_negative = numpy.all(a >= 0)
    normalized = numpy.isclose(sum(a), 1.0)
    return not_negative and normalized
<commit_msg>check alpha in staticmethods<commit_after># -*- coding: utf-8 -*-
# Copyright (C) 2016,2017 Simone Marsili
# All rights reserved.
# License: BSD 3 clause
"""Base classes module."""
from __future__ import (absolute_import, division,
                        print_function, unicode_literals)
from builtins import (  # pylint: disable=redefined-builtin, unused-import
    bytes, dict, int, list, object, range, str,
    ascii, chr, hex, input, next, oct, open,
    pow, round, super,
    filter, map, zip)
import numpy
from ndd.base import BaseEstimator
import ndd.fnsb

class EntropyEstimatorMixin(object):
    @staticmethod
    def _plugin_estimator(pk, k):
        return ndd.fnsb.plugin(pk, k), None

    @staticmethod
    def _pseudocounts_estimator(pk, k, alpha):
        try:
            alpha = numpy.float64(alpha)
        except ValueError:
            raise ValueError('alpha (%r) should be numeric.' % alpha)
        if alpha < 0:
            raise ValueError('Negative alpha value: %r' % alpha)
        return ndd.fnsb.pseudo(pk, k, alpha), None

    @staticmethod
    def _ww_estimator(pk, k, alpha):
        try:
            alpha = numpy.float64(alpha)
        except ValueError:
            raise ValueError('alpha (%r) should be numeric.' % alpha)
        if alpha < 0:
            raise ValueError('Negative alpha value: %r' % alpha)
        return ndd.fnsb.dirichlet(pk, k, alpha), None

    @staticmethod
    def _nsb_estimator(pk, k):
        return ndd.fnsb.nsb(pk, k)

    def estimator(self, pk, k):
        if self._estimator is None:
            if self.plugin:
                if self.alpha is None:
                    self._estimator = self._plugin_estimator
                else:
                    self._estimator = lambda pk, k: self._pseudocounts_estimator(pk, k, self.alpha)  # pylint: disable=redefined-variable-type
            else:
                if self.alpha is None:
                    self._estimator = self._nsb_estimator
                else:
                    self._estimator = lambda pk, k: self._ww_estimator(pk, k, self.alpha)
        return self._estimator(pk, k)

    @staticmethod
    def _check_k(k, n_bins):
        """
        if k is None, set k = number of bins
        if k is an integer, just check
        ik an array set k = prod(k)
        """
        MAX_LOGK = 150 * numpy.log(2)

        if k is None:
            # set k to the number of observed bins
            k = numpy.float64(n_bins)
        else:
            try:
                k = numpy.float64(k)
            except ValueError:
                raise
            if k.ndim:
                # if k is a sequence, set k = prod(k)
                if k.ndim > 1:
                    raise ValueError('k must be a scalar or 1D array')
                logk = numpy.sum(numpy.log(x) for x in k)
                if logk > MAX_LOGK:
                    # too large a number; backoff to n_bins?
                    # TODO: log warning
                    raise ValueError('k (%r) larger than %r' %
                                     (numpy.exp(logk), numpy.exp(MAX_LOGK)))
                else:
                    k = numpy.prod(k)
            else:
                # if a scalar check size
                if numpy.log(k) > MAX_LOGK:
                    raise ValueError('k (%r) larger than %r' %
                                     (k, numpy.exp(MAX_LOGK)))
            # consistency checks
            if k < n_bins:
                raise ValueError("k (%s) is smaller than the number of bins"
                                 "(%s)" % (k, n_bins))
            if not k.is_integer():
                raise ValueError("k (%s) should be a whole number." % k)
        return k

# TODO: docstrings
class Entropy(EntropyEstimatorMixin, BaseEstimator):
    def __init__(self, alpha=None, plugin=False):

        self.estimate = None
        self.std = None
        self._estimator = None

        self.alpha = alpha
        self.plugin = plugin

    def _check_input(self, pk, k):
        pk = self._check_counts(a=pk)
        k = self._check_k(k=k, n_bins=len(pk))
        return pk, k

    @staticmethod
    def _check_counts(a):
        a = numpy.float64(a).flatten()
        not_integers = not numpy.all([x.is_integer() for x in a])
        negative = numpy.any([a < 0])
        if not_integers:
            raise ValueError('counts array has non-integer values')
        if negative:
            raise ValueError('counts array has negative values')
        return numpy.int32(a)

    def fit(self, pk, k=None):
        """
        pk : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(pk).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(pk).

        """
        pk, k = self._check_input(pk, k)
        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            self.estimate, self.std = self.estimator(pk, k)

    def __call__(self, *args, **kwargs):
        """Return estimate from input data. Delegate to fit."""
        self.fit(*args, **kwargs)
        return self.estimate

class KLDivergence(Entropy):
    def __init__(self, qk, alpha=None, plugin=False):
        """qk is a parameter of the estimator; it must be a valid pmf."""
        super().__init__(alpha, plugin)
        if is_pmf(qk):
            self.log_qk = numpy.log(qk)
        else:
            raise ValueError('qk must be a valid PMF')

    def fit(self, pk, k=None):
        """
        pk : array_like
            The number of occurrences of a set of bins.

        k : int, optional
            Number of bins. k >= len(pk).
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to len(pk).

        """
        pk, k = self._check_input(pk, k)
        if len(self.log_qk) != len(pk):
            raise ValueError('qk and pk must have the same length.')

        if k == 1:  # single bin
            self.estimate = self.std = 0.0
        else:
            self.estimate, self.std = self.estimator(pk, k)
        self.estimate -= numpy.sum(pk * self.log_qk)

class JSDivergence(Entropy):
    @staticmethod
    def _check_counts(a):
        a = numpy.float64(a)
        if a.ndim != 2:
            raise ValueError('counts must be 2D.')
        not_integers = not numpy.all([x.is_integer() for x in a.flatten()])
        negative = numpy.any([a < 0])
        if not_integers:
            raise ValueError('counts array has non-integer values')
        if negative:
            raise ValueError('counts array has negative values')
        return numpy.int32(a)

    def _check_input(self, pk, k):
        pk = self._check_counts(a=pk)
        k = self._check_k(k=k, n_bins=pk.shape[1])
        return pk, k

    def fit(self, pk, k=None):
        """
        pk : array_like
            n-by-p array. Different rows correspond to counts from different
            distributions with the same discrete sample space.

        k : int, optional
            Number of bins. k >= p if pk is n-by-p.
            Float values are valid input for whole numbers (e.g. k=1.e3).
            Defaults to pk.shape[1].

        """
        pk, k = self._check_input(pk, k)
        ws = numpy.float64(pk.sum(axis=1))
        ws /= ws.sum()
        if k == 1:  # single bin
            self.estimate = 0.0
        else:
            self.estimate = self.estimator(pk.sum(axis=0), k)[0] - sum(
                ws[i]*self.estimator(x, k)[0] for i, x in enumerate(pk))

def _test_JS(n):
    alpha = 0.1
    p = 100
    p1 = numpy.random.dirichlet([alpha]*p)
    p2 = numpy.random.dirichlet([alpha]*p)
    pm = 0.5 * (p1 + p2)

    def ee(x):
        y = - x * numpy.log(x)
        return numpy.sum(y[x > 0])
    js = ee(pm) - 0.5 * (ee(p1) + ee(p2))
    c1 = numpy.random.multinomial(n, p1)
    c2 = numpy.random.multinomial(n, p2)
    pk = numpy.stack([c1, c2])
    est = JSDivergence()
    est.fit(pk)
    js1 = est.estimate
    return js, js1

def is_pmf(a):
    a = numpy.float64(a)
    not_negative = numpy.all(a >= 0)
    normalized = numpy.isclose(sum(a), 1.0)
    return not_negative and normalized
<|endoftext|>


<commit_before>import numpy as np
import itertools
import random
import math

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip(a, b)

def logistic(x):
    return np.tanh(x)
logistic.derivative = lambda x : (1-np.square(x))

class NeuralNetwork:

    def __init__(self, weights, bias=None, sigmoid=None):
        self.weights = weights
        if sigmoid is None:
            sigmoid = logistic
        self.sigmoid = sigmoid
        self.bias = bias

    @classmethod
    def fromlayers(cls, sizes, bias=True, sigmoid=None):
        weights = [2*np.matrix(np.random.rand(b,a))-1
                   for a,b in pairwise(sizes)]
        if bias:
            bias = [2*np.matrix(np.random.rand(b,1))-1
                    for a,b in pairwise(sizes)]
            return cls(weights, bias, sigmoid)
        return cls(weights, sigmoid)

    def copy(self):
        return NeuralNetwork(
            [np.matrix(x) for x in self.weights],
            ([np.matrix(x) for x in self.bias]
             if self.bias is not None else None),
            self.sigmoid
            )

    def mutate(self, scale=1):
        for weight_matrix in self.weights:
            weight_matrix += (
                2*scale *
                (np.random.rand(*weight_matrix.shape)-0.5))
        if self.bias is not None:
            for bias in self.bias:
                bias += (
                    2*scale *
                    (np.random.rand(*bias.shape)-0.5))

    def __call__(self, potentials, trace=False):
        potentials = np.matrix(potentials, dtype=np.float64
                               ).reshape(len(potentials), 1)

        if trace:
            trace = [potentials]

        for index, weight_matrix in enumerate(self.weights):
            potentials = weight_matrix * potentials
            if self.bias is not None:
                potentials += self.bias[index]
            potentials = self.sigmoid(potentials)
            if trace:
                trace.append(potentials)

        if trace:
            return trace
        return potentials

    def __repr__(self):
        return ("weights:\n"+
                "\n".join(map(repr,self.weights))+"\n"
                "bias:\n"+
                "\n".join(map(repr, self.bias)))

def backpropagate(network, tests, iterations=50):
    for _ in range(iterations):
        traces = [(network(test[0], True), test[1]) for test in tests]
        errors =[sum(expected-trace[-1] for trace,expected in traces)]
        #trace = np.sum(trace[0] for trace in traces)
        print("\n".join(map(str,traces)))
        print('error')
        print(errors)
        #print(trace)
        for index, weights in reversed(
            list(enumerate(network.weights))):

            print(index)
            error = errors[-1]

if True:
    global network
    tests = [((0,0),0),((0,1),1),((1,0),1),((1,1),0)]
    network = NeuralNetwork.fromlayers([2,3,1])
    backpropagate(network, tests, 1)
<commit_msg>added comments<commit_after>import numpy as np
import itertools
import random
import math

def pairwise(iterable):
    "s -> (s0,s1), (s1,s2), (s2, s3), ..."
    a, b = itertools.tee(iterable)
    next(b, None)
    return zip(a, b)

def logistic(x):
    return np.tanh(x)
logistic.derivative = lambda x : (1-np.square(x))

class NeuralNetwork:

    def __init__(self, weights, bias=None, sigmoid=None):
        self.weights = weights
        if sigmoid is None:
            sigmoid = logistic
        self.sigmoid = sigmoid
        self.bias = bias

    @classmethod
    def fromlayers(cls, sizes, bias=True, sigmoid=None):
        weights = [2*np.matrix(np.random.rand(b,a))-1
                   for a,b in pairwise(sizes)]
        if bias:
            bias = [2*np.matrix(np.random.rand(b,1))-1
                    for a,b in pairwise(sizes)]
            return cls(weights, bias, sigmoid)
        return cls(weights, sigmoid)

    def copy(self):
        return NeuralNetwork(
            [np.matrix(x) for x in self.weights],
            ([np.matrix(x) for x in self.bias]
             if self.bias is not None else None),
            self.sigmoid
            )

    def mutate(self, scale=1):
        for weight_matrix in self.weights:
            weight_matrix += (
                2*scale *
                (np.random.rand(*weight_matrix.shape)-0.5))
        if self.bias is not None:
            for bias in self.bias:
                bias += (
                    2*scale *
                    (np.random.rand(*bias.shape)-0.5))

    def __call__(self, potentials, trace=False):
        potentials = np.matrix(potentials, dtype=np.float64
                               ).reshape(len(potentials), 1)

        if trace:
            trace = [potentials]

        for index, weight_matrix in enumerate(self.weights):
            potentials = weight_matrix * potentials
            if self.bias is not None:
                potentials += self.bias[index]
            potentials = self.sigmoid(potentials)
            if trace:
                trace.append(potentials)

        if trace:
            return trace
        return potentials

    def __repr__(self):
        return ("weights:\n"+
                "\n".join(map(repr,self.weights))+"\n"
                "bias:\n"+
                "\n".join(map(repr, self.bias)))

def backpropagate(network, tests, iterations=50):

    #convert tests into numpy matrices
    tests = [(np.matrix(inputs, dtype=np.float64).reshape(len(inputs), 1),
            np.matrix(expected, dtype=np.float64).reshape(len(expected), 1))
            for inputs, expected in tests]

    for _ in range(iterations):

        #accumulate the weight and bias deltas
        weight_delta = [np.zeros(matrix.shape) for matrix in network.weights]
        bias_delta = [np.zeros(matrix.shape) for matrix in network.bias]

        #iterate over the tests
        for potentials, expected in tests:

            #input the potentials into the network
            #calling the network with trace == True returns a list of matrices,
            #representing the potentials of each layer
            trace = network(potentials, trace=True)
            errors = [expected - trace[-1]]

            #iterate over the layers backwards
            for weight_matrix, layer in reversed(list(zip(network.weights, trace))):
                #compute the error vector for a layer
                errors.append(np.multiply(weight_matrix.transpose()*errors[-1],
                                          network.sigmoid.derivative(layer)))

            #remove the input layer
            errors.pop()
            errors.reverse()

            #compute the deltas for bias and weight
            for index, error in enumerate(errors):
                bias_delta[index] += error
                weight_delta[index] += error * trace[index].transpose()

        #apply the deltas
        for index, delta in enumerate(weight_delta):
            network.weights[index] += delta
        for index, delta in enumerate(bias_delta):
            network.bias[index] += delta

if True:
    global network
    tests = [((0,0),[0]),((0,1),[1]),((1,0),[1]),((1,1),[0])]
    network = NeuralNetwork.fromlayers([2,5,1])
    backpropagate(network, tests, 500)
    for test in tests:
        print(test[0])
        print(str(network(test[0])) + str(test[1]))
<|endoftext|>


<commit_before>import re
from setuptools import setup

def find_version(filename):
    _version_re = re.compile(r"__version__ = '(.*)'")
    for line in open(filename):
        version_match = _version_re.match(line)
        if version_match:
            return version_match.group(1)

__version__ = find_version('neurondm/__init__.py')

with open('README.md', 'rt') as f:
    long_description = f.read()

tests_require = ['pytest', 'pytest-runner']
setup(
    name='neurondm',
    version=__version__,
    description='A data model for neuron types.',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/tgbugs/pyontutils/tree/master/neurondm',
    author='Tom Gillespie',
    author_email='tgbugs@gmail.com',
    license='MIT',
    classifiers=[
        'Development Status :: 4 - Beta',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
    ],
    keywords=('neuron types NIF ontology neuroscience phenotype '
              'OWL rdf rdflib data model'),
    packages=['neurondm'],  # don't package models due to data resources needs?
    python_requires='>=3.6',
    tests_require=tests_require,
    install_requires=[
        'pyontutils>=0.1.1',
    ],
    extras_require={'dev': ['pytest-cov', 'wheel'],
                    'test': tests_require,
                    'notebook': ['jupyter'],
    },
    entry_points={
        'console_scripts': [
        ],
    },
)
<commit_msg>neurondm setup.py update pyontutils version requirement<commit_after>import re
from setuptools import setup

def find_version(filename):
    _version_re = re.compile(r"__version__ = '(.*)'")
    for line in open(filename):
        version_match = _version_re.match(line)
        if version_match:
            return version_match.group(1)

__version__ = find_version('neurondm/__init__.py')

with open('README.md', 'rt') as f:
    long_description = f.read()

tests_require = ['pytest', 'pytest-runner']
setup(
    name='neurondm',
    version=__version__,
    description='A data model for neuron types.',
    long_description=long_description,
    long_description_content_type='text/markdown',
    url='https://github.com/tgbugs/pyontutils/tree/master/neurondm',
    author='Tom Gillespie',
    author_email='tgbugs@gmail.com',
    license='MIT',
    classifiers=[
        'Development Status :: 4 - Beta',
        'License :: OSI Approved :: MIT License',
        'Programming Language :: Python :: 3.6',
        'Programming Language :: Python :: 3.7',
    ],
    keywords=('neuron types NIF ontology neuroscience phenotype '
              'OWL rdf rdflib data model'),
    packages=['neurondm'],  # don't package models due to data resources needs?
    python_requires='>=3.6',
    tests_require=tests_require,
    install_requires=[
        'pyontutils>=0.1.2',
    ],
    extras_require={'dev': ['pytest-cov', 'wheel'],
                    'test': tests_require,
                    'notebook': ['jupyter'],
    },
    entry_points={
        'console_scripts': [
        ],
    },
)
<|endoftext|>


<commit_before>"""Django Ninja - Fast Django REST framework"""

__version__ = "0.16.1"

from pydantic import Field

from ninja.files import UploadedFile
from ninja.main import NinjaAPI
from ninja.orm import ModelSchema
from ninja.params_functions import Body, Cookie, File, Form, Header, Path, Query
from ninja.router import Router
from ninja.schema import Schema

__all__ = [
    "Field",
    "UploadedFile",
    "NinjaAPI",
    "Body",
    "Cookie",
    "File",
    "Form",
    "Header",
    "Path",
    "Query",
    "Router",
    "Schema",
    "ModelSchema",
]
<commit_msg>0.16.2<commit_after>"""Django Ninja - Fast Django REST framework"""

__version__ = "0.16.2"

from pydantic import Field

from ninja.files import UploadedFile
from ninja.main import NinjaAPI
from ninja.orm import ModelSchema
from ninja.params_functions import Body, Cookie, File, Form, Header, Path, Query
from ninja.router import Router
from ninja.schema import Schema

__all__ = [
    "Field",
    "UploadedFile",
    "NinjaAPI",
    "Body",
    "Cookie",
    "File",
    "Form",
    "Header",
    "Path",
    "Query",
    "Router",
    "Schema",
    "ModelSchema",
]
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
:copyright: (c) 2014-2016 by Mike Taylor
:license: MIT, see LICENSE for more details.
"""
from .indieauth import discoverAuthEndpoints, validateAuthCode              # noqa
from .micropub import discoverMicropubEndpoints, discoverTokenEndpoints     # noqa

__author__       = 'bear (Mike Taylor)'
__email__        = 'bear@bear.im'
__copyright__    = 'Copyright (c) 2014-2016 by Mike Taylor'
__license__      = 'MIT'
__version__      = '0.3.3'
__url__          = 'https://github.com/bear/ninka'
__download_url__ = 'https://pypi.python.org/pypi/ninka'
__description__  = "Indieauth Toolkit"
<commit_msg>bump version for release<commit_after># -*- coding: utf-8 -*-
"""
:copyright: (c) 2014-2016 by Mike Taylor
:license: MIT, see LICENSE for more details.
"""
from .indieauth import discoverAuthEndpoints, validateAuthCode              # noqa
from .micropub import discoverMicropubEndpoints, discoverTokenEndpoints     # noqa

__author__       = 'bear (Mike Taylor)'
__email__        = 'bear@bear.im'
__copyright__    = 'Copyright (c) 2014-2016 by Mike Taylor'
__license__      = 'MIT'
__version__      = '0.3.4'
__url__          = 'https://github.com/bear/ninka'
__download_url__ = 'https://pypi.python.org/pypi/ninka'
__description__  = "Indieauth Toolkit"
<|endoftext|>