<commit_before>"""Capture C-level FD output on pipes

Use `wurlitzer.pipes` or `wurlitzer.sys_pipes` as context managers.
"""
from __future__ import print_function

__version__ = '1.0.3.dev'

__all__ = [
    'pipes',
    'sys_pipes',
    'sys_pipes_forever',
    'stop_sys_pipes',
    'Wurlitzer',
]

from contextlib import contextmanager
import ctypes
import errno
from fcntl import fcntl, F_GETFL, F_SETFL
import io
import os
try:
    from queue import Queue
except ImportError:
    from Queue import Queue
import select
import sys
import threading
import time
import warnings

libc = ctypes.CDLL(None)

try:
    c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout')
    c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr')
except ValueError: # pragma: no cover
    # libc.stdout is has a funny name on OS X
    c_stdout_p = ctypes.c_void_p.in_dll(libc, '__stdoutp') # pragma: no cover
    c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp') # pragma: no cover

STDOUT = 2
PIPE = 3

_default_encoding = getattr(sys.stdin, 'encoding', None) or 'utf8'
if _default_encoding.lower() == 'ascii':
    # don't respect ascii
    _default_encoding = 'utf8' # pragma: no cover

def dup2(a, b, timeout=3):
    """Like os.dup2, but retry on EBUSY"""
    dup_err = None
    # give FDs 3 seconds to not be busy anymore
    for i in range(int(10 * timeout)):
        try:
            return os.dup2(a, b)
        except OSError as e:
            dup_err = e
            if e.errno == errno.EBUSY:
                time.sleep(0.1)
            else:
                raise
    if dup_err:
        raise dup_err

class Wurlitzer(object):
    """Class for Capturing Process-level FD output via dup2

    Typically used via `wurlitzer.capture`
    """
    flush_interval = 0.2

    def __init__(self, stdout=None, stderr=None, encoding=_default_encoding):
        """
        Parameters
        ----------
        stdout: stream or None
            The stream for forwarding stdout.
        stderr = stream or None
            The stream for forwarding stderr.
        encoding: str or None
            The encoding to use, if streams should be interpreted as text.
        """
        self._stdout = stdout
        if stderr == STDOUT:
            self._stderr = self._stdout
        else:
            self._stderr = stderr
        self.encoding = encoding
        self._save_fds = {}
        self._real_fds = {}
        self._handlers = {}
        self._handlers['stderr'] = self._handle_stderr
        self._handlers['stdout'] = self._handle_stdout

    def _setup_pipe(self, name):
        real_fd = getattr(sys, '__%s__' % name).fileno()
        save_fd = os.dup(real_fd)
        self._save_fds[name] = save_fd

        pipe_out, pipe_in = os.pipe()
        dup2(pipe_in, real_fd)
        os.close(pipe_in)
        self._real_fds[name] = real_fd

        # make pipe_out non-blocking
        flags = fcntl(pipe_out, F_GETFL)
        fcntl(pipe_out, F_SETFL, flags|os.O_NONBLOCK)
        return pipe_out

    def _decode(self, data):
        """Decode data, if any

        Called before passing to stdout/stderr streams
        """
        if self.encoding:
            data = data.decode(self.encoding, 'replace')
        return data

    def _handle_stdout(self, data):
        if self._stdout:
            self._stdout.write(self._decode(data))

    def _handle_stderr(self, data):
        if self._stderr:
            self._stderr.write(self._decode(data))

    def _setup_handle(self):
        """Setup handle for output, if any"""
        self.handle = (self._stdout, self._stderr)

    def _finish_handle(self):
        """Finish handle, if anything should be done when it's all wrapped up."""
        pass

    def __enter__(self):
        # flush anything out before starting
        libc.fflush(c_stdout_p)
        libc.fflush(c_stderr_p)
        # setup handle
        self._setup_handle()
        self._control_r, self._control_w = os.pipe()

        # create pipe for stdout
        pipes = [self._control_r]
        names = {self._control_r: 'control'}
        if self._stdout:
            pipe = self._setup_pipe('stdout')
            pipes.append(pipe)
            names[pipe] = 'stdout'
        if self._stderr:
            pipe = self._setup_pipe('stderr')
            pipes.append(pipe)
            names[pipe] = 'stderr'

        # flush pipes in a background thread to avoid blocking
        # the reader thread when the buffer is full
        flush_queue = Queue()

        def flush_main():
            while True:
                msg = flush_queue.get()
                if msg == 'stop':
                    return
                libc.fflush(c_stdout_p)
                libc.fflush(c_stderr_p)

        flush_thread = threading.Thread(target=flush_main)
        flush_thread.daemon = True
        flush_thread.start()

        def forwarder():
            """Forward bytes on a pipe to stream messages"""
            draining = False
            flush_interval = 0
            poller = select.poll()
            for pipe_ in pipes: poller.register(pipe_)

            while pipes:
                events = poller.poll(flush_interval)
                #r = all([(r_[1] == (select.POLLIN | select.POLLPRI)) for r_ in events])
                if events:
                    # found something to read, don't block select until
                    # we run out of things to read
                    flush_interval = 0
                else:
                    # nothing to read
                    if draining:
                        # if we are draining and there's nothing to read, stop
                        break
                    else:
                        # nothing to read, get ready to wait.
                        # flush the streams in case there's something waiting
                        # to be written.
                        flush_queue.put('flush')
                        flush_interval = self.flush_interval
                        continue

                for fd, flags in events:
                    if fd == self._control_r:
                        draining = True
                        os.close(self._control_r)
                        pipes.remove(self._control_r)
                        poller.unregister(self._control_r)
                        continue
                    name = names[fd]
                    data = os.read(fd, 1024)
                    if not data:
                        # pipe closed, stop polling it
                        pipes.remove(fd)
                        poller.unregister(fd)
                        os.close(fd)
                    else:
                        handler = getattr(self, '_handle_%s' % name)
                        handler(data)
                if not pipes:
                    # pipes closed, we are done
                    break
            # stop flush thread
            flush_queue.put('stop')
            flush_thread.join()
            # cleanup pipes
            [os.close(pipe) for pipe in pipes]

        self.thread = threading.Thread(target=forwarder)
        self.thread.daemon = True
        self.thread.start()

        return self.handle

    def __exit__(self, exc_type, exc_value, traceback):
        # flush the underlying C buffers
        libc.fflush(c_stdout_p)
        libc.fflush(c_stderr_p)
        # signal output is complete on control pipe
        os.write(self._control_w, b'\1')
        self.thread.join()
        os.close(self._control_w)

        # restore original state
        for name, real_fd in self._real_fds.items():
            save_fd = self._save_fds[name]
            dup2(save_fd, real_fd)
            os.close(save_fd)
        # finalize handle
        self._finish_handle()

@contextmanager
def pipes(stdout=PIPE, stderr=PIPE, encoding=_default_encoding):
    """Capture C-level stdout/stderr in a context manager.

    The return value for the context manager is (stdout, stderr).

    Examples
    --------

    >>> with capture() as (stdout, stderr):
    ...     printf("C-level stdout")
    ... output = stdout.read()
    """
    stdout_pipe = stderr_pipe = False
    # setup stdout
    if stdout == PIPE:
        stdout_r, stdout_w = os.pipe()
        stdout_w = os.fdopen(stdout_w, 'wb')
        if encoding:
            stdout_r = io.open(stdout_r, 'r', encoding=encoding)
        else:
            stdout_r = os.fdopen(stdout_r, 'rb')
        stdout_pipe = True
    else:
        stdout_r = stdout_w = stdout
    # setup stderr
    if stderr == STDOUT:
        stderr_r = None
        stderr_w = stdout_w
    elif stderr == PIPE:
        stderr_r, stderr_w = os.pipe()
        stderr_w = os.fdopen(stderr_w, 'wb')
        if encoding:
            stderr_r = io.open(stderr_r, 'r', encoding=encoding)
        else:
            stderr_r = os.fdopen(stderr_r, 'rb')
        stderr_pipe = True
    else:
        stderr_r = stderr_w = stderr
    if stdout_pipe or stderr_pipe:
        capture_encoding = None
    else:
        capture_encoding = encoding
    w = Wurlitzer(stdout=stdout_w, stderr=stderr_w, encoding=capture_encoding)
    try:
        with w:
            yield stdout_r, stderr_r
    finally:
        # close pipes
        if stdout_pipe:
            stdout_w.close()
        if stderr_pipe:
            stderr_w.close()

def sys_pipes(encoding=_default_encoding):
    """Redirect C-level stdout/stderr to sys.stdout/stderr

    This is useful of sys.sdout/stderr are already being forwarded somewhere.

    DO NOT USE THIS if sys.stdout and sys.stderr are not already being forwarded.
    """
    return pipes(sys.stdout, sys.stderr, encoding=encoding)

_mighty_wurlitzer = None

def sys_pipes_forever(encoding=_default_encoding):
    """Redirect all C output to sys.stdout/err

    This is not a context manager; it turns on C-forwarding permanently.
    """
    global _mighty_wurlitzer
    if _mighty_wurlitzer is None:
        _mighty_wurlitzer = sys_pipes(encoding)
    _mighty_wurlitzer.__enter__()

def stop_sys_pipes():
    """Stop permanent redirection started by sys_pipes_forever"""
    global _mighty_wurlitzer
    if _mighty_wurlitzer is not None:
        _mighty_wurlitzer.__exit__(None, None, None)
        _mighty_wurlitzer = None

def load_ipython_extension(ip):
    """Register me as an IPython extension

    Captures all C output during execution and forwards to sys.

    Does nothing on terminal IPython.

    Use: %load_ext wurlitzer
    """
    if not getattr(ip, 'kernel'):
        warnings.warn(
            "wurlitzer extension doesn't do anything in terminal IPython"
        )
        return
    ip.events.register('pre_execute', sys_pipes_forever)
    ip.events.register('post_execute', stop_sys_pipes)

def unload_ipython_extension(ip):
    """Unload me as an IPython extension

    Use: %unload_ext wurlitzer
    """
    if not getattr(ip, 'kernel'):
        return
    ip.events.unregister('pre_execute', sys_pipes_forever)
    ip.events.unregister('post_execute', stop_sys_pipes)
<commit_msg>closing socket descriptor after unregistering from poller for control r<commit_after>"""Capture C-level FD output on pipes

Use `wurlitzer.pipes` or `wurlitzer.sys_pipes` as context managers.
"""
from __future__ import print_function

__version__ = '1.0.3.dev'

__all__ = [
    'pipes',
    'sys_pipes',
    'sys_pipes_forever',
    'stop_sys_pipes',
    'Wurlitzer',
]

from contextlib import contextmanager
import ctypes
import errno
from fcntl import fcntl, F_GETFL, F_SETFL
import io
import os
try:
    from queue import Queue
except ImportError:
    from Queue import Queue
import select
import sys
import threading
import time
import warnings

libc = ctypes.CDLL(None)

try:
    c_stdout_p = ctypes.c_void_p.in_dll(libc, 'stdout')
    c_stderr_p = ctypes.c_void_p.in_dll(libc, 'stderr')
except ValueError: # pragma: no cover
    # libc.stdout is has a funny name on OS X
    c_stdout_p = ctypes.c_void_p.in_dll(libc, '__stdoutp') # pragma: no cover
    c_stderr_p = ctypes.c_void_p.in_dll(libc, '__stderrp') # pragma: no cover

STDOUT = 2
PIPE = 3

_default_encoding = getattr(sys.stdin, 'encoding', None) or 'utf8'
if _default_encoding.lower() == 'ascii':
    # don't respect ascii
    _default_encoding = 'utf8' # pragma: no cover

def dup2(a, b, timeout=3):
    """Like os.dup2, but retry on EBUSY"""
    dup_err = None
    # give FDs 3 seconds to not be busy anymore
    for i in range(int(10 * timeout)):
        try:
            return os.dup2(a, b)
        except OSError as e:
            dup_err = e
            if e.errno == errno.EBUSY:
                time.sleep(0.1)
            else:
                raise
    if dup_err:
        raise dup_err

class Wurlitzer(object):
    """Class for Capturing Process-level FD output via dup2

    Typically used via `wurlitzer.capture`
    """
    flush_interval = 0.2

    def __init__(self, stdout=None, stderr=None, encoding=_default_encoding):
        """
        Parameters
        ----------
        stdout: stream or None
            The stream for forwarding stdout.
        stderr = stream or None
            The stream for forwarding stderr.
        encoding: str or None
            The encoding to use, if streams should be interpreted as text.
        """
        self._stdout = stdout
        if stderr == STDOUT:
            self._stderr = self._stdout
        else:
            self._stderr = stderr
        self.encoding = encoding
        self._save_fds = {}
        self._real_fds = {}
        self._handlers = {}
        self._handlers['stderr'] = self._handle_stderr
        self._handlers['stdout'] = self._handle_stdout

    def _setup_pipe(self, name):
        real_fd = getattr(sys, '__%s__' % name).fileno()
        save_fd = os.dup(real_fd)
        self._save_fds[name] = save_fd

        pipe_out, pipe_in = os.pipe()
        dup2(pipe_in, real_fd)
        os.close(pipe_in)
        self._real_fds[name] = real_fd

        # make pipe_out non-blocking
        flags = fcntl(pipe_out, F_GETFL)
        fcntl(pipe_out, F_SETFL, flags|os.O_NONBLOCK)
        return pipe_out

    def _decode(self, data):
        """Decode data, if any

        Called before passing to stdout/stderr streams
        """
        if self.encoding:
            data = data.decode(self.encoding, 'replace')
        return data

    def _handle_stdout(self, data):
        if self._stdout:
            self._stdout.write(self._decode(data))

    def _handle_stderr(self, data):
        if self._stderr:
            self._stderr.write(self._decode(data))

    def _setup_handle(self):
        """Setup handle for output, if any"""
        self.handle = (self._stdout, self._stderr)

    def _finish_handle(self):
        """Finish handle, if anything should be done when it's all wrapped up."""
        pass

    def __enter__(self):
        # flush anything out before starting
        libc.fflush(c_stdout_p)
        libc.fflush(c_stderr_p)
        # setup handle
        self._setup_handle()
        self._control_r, self._control_w = os.pipe()

        # create pipe for stdout
        pipes = [self._control_r]
        names = {self._control_r: 'control'}
        if self._stdout:
            pipe = self._setup_pipe('stdout')
            pipes.append(pipe)
            names[pipe] = 'stdout'
        if self._stderr:
            pipe = self._setup_pipe('stderr')
            pipes.append(pipe)
            names[pipe] = 'stderr'

        # flush pipes in a background thread to avoid blocking
        # the reader thread when the buffer is full
        flush_queue = Queue()

        def flush_main():
            while True:
                msg = flush_queue.get()
                if msg == 'stop':
                    return
                libc.fflush(c_stdout_p)
                libc.fflush(c_stderr_p)

        flush_thread = threading.Thread(target=flush_main)
        flush_thread.daemon = True
        flush_thread.start()

        def forwarder():
            """Forward bytes on a pipe to stream messages"""
            draining = False
            flush_interval = 0
            poller = select.poll()
            for pipe_ in pipes: poller.register(pipe_)

            while pipes:
                events = poller.poll(flush_interval)
                #r = all([(r_[1] == (select.POLLIN | select.POLLPRI)) for r_ in events])
                if events:
                    # found something to read, don't block select until
                    # we run out of things to read
                    flush_interval = 0
                else:
                    # nothing to read
                    if draining:
                        # if we are draining and there's nothing to read, stop
                        break
                    else:
                        # nothing to read, get ready to wait.
                        # flush the streams in case there's something waiting
                        # to be written.
                        flush_queue.put('flush')
                        flush_interval = self.flush_interval
                        continue

                for fd, flags in events:
                    if fd == self._control_r:
                        draining = True
                        pipes.remove(self._control_r)
                        poller.unregister(self._control_r)
                        os.close(self._control_r)
                        continue
                    name = names[fd]
                    data = os.read(fd, 1024)
                    if not data:
                        # pipe closed, stop polling it
                        pipes.remove(fd)
                        poller.unregister(fd)
                        os.close(fd)
                    else:
                        handler = getattr(self, '_handle_%s' % name)
                        handler(data)
                if not pipes:
                    # pipes closed, we are done
                    break
            # stop flush thread
            flush_queue.put('stop')
            flush_thread.join()
            # cleanup pipes
            [os.close(pipe) for pipe in pipes]

        self.thread = threading.Thread(target=forwarder)
        self.thread.daemon = True
        self.thread.start()

        return self.handle

    def __exit__(self, exc_type, exc_value, traceback):
        # flush the underlying C buffers
        libc.fflush(c_stdout_p)
        libc.fflush(c_stderr_p)
        # signal output is complete on control pipe
        os.write(self._control_w, b'\1')
        self.thread.join()
        os.close(self._control_w)

        # restore original state
        for name, real_fd in self._real_fds.items():
            save_fd = self._save_fds[name]
            dup2(save_fd, real_fd)
            os.close(save_fd)
        # finalize handle
        self._finish_handle()

@contextmanager
def pipes(stdout=PIPE, stderr=PIPE, encoding=_default_encoding):
    """Capture C-level stdout/stderr in a context manager.

    The return value for the context manager is (stdout, stderr).

    Examples
    --------

    >>> with capture() as (stdout, stderr):
    ...     printf("C-level stdout")
    ... output = stdout.read()
    """
    stdout_pipe = stderr_pipe = False
    # setup stdout
    if stdout == PIPE:
        stdout_r, stdout_w = os.pipe()
        stdout_w = os.fdopen(stdout_w, 'wb')
        if encoding:
            stdout_r = io.open(stdout_r, 'r', encoding=encoding)
        else:
            stdout_r = os.fdopen(stdout_r, 'rb')
        stdout_pipe = True
    else:
        stdout_r = stdout_w = stdout
    # setup stderr
    if stderr == STDOUT:
        stderr_r = None
        stderr_w = stdout_w
    elif stderr == PIPE:
        stderr_r, stderr_w = os.pipe()
        stderr_w = os.fdopen(stderr_w, 'wb')
        if encoding:
            stderr_r = io.open(stderr_r, 'r', encoding=encoding)
        else:
            stderr_r = os.fdopen(stderr_r, 'rb')
        stderr_pipe = True
    else:
        stderr_r = stderr_w = stderr
    if stdout_pipe or stderr_pipe:
        capture_encoding = None
    else:
        capture_encoding = encoding
    w = Wurlitzer(stdout=stdout_w, stderr=stderr_w, encoding=capture_encoding)
    try:
        with w:
            yield stdout_r, stderr_r
    finally:
        # close pipes
        if stdout_pipe:
            stdout_w.close()
        if stderr_pipe:
            stderr_w.close()

def sys_pipes(encoding=_default_encoding):
    """Redirect C-level stdout/stderr to sys.stdout/stderr

    This is useful of sys.sdout/stderr are already being forwarded somewhere.

    DO NOT USE THIS if sys.stdout and sys.stderr are not already being forwarded.
    """
    return pipes(sys.stdout, sys.stderr, encoding=encoding)

_mighty_wurlitzer = None

def sys_pipes_forever(encoding=_default_encoding):
    """Redirect all C output to sys.stdout/err

    This is not a context manager; it turns on C-forwarding permanently.
    """
    global _mighty_wurlitzer
    if _mighty_wurlitzer is None:
        _mighty_wurlitzer = sys_pipes(encoding)
    _mighty_wurlitzer.__enter__()

def stop_sys_pipes():
    """Stop permanent redirection started by sys_pipes_forever"""
    global _mighty_wurlitzer
    if _mighty_wurlitzer is not None:
        _mighty_wurlitzer.__exit__(None, None, None)
        _mighty_wurlitzer = None

def load_ipython_extension(ip):
    """Register me as an IPython extension

    Captures all C output during execution and forwards to sys.

    Does nothing on terminal IPython.

    Use: %load_ext wurlitzer
    """
    if not getattr(ip, 'kernel'):
        warnings.warn(
            "wurlitzer extension doesn't do anything in terminal IPython"
        )
        return
    ip.events.register('pre_execute', sys_pipes_forever)
    ip.events.register('post_execute', stop_sys_pipes)

def unload_ipython_extension(ip):
    """Unload me as an IPython extension

    Use: %unload_ext wurlitzer
    """
    if not getattr(ip, 'kernel'):
        return
    ip.events.unregister('pre_execute', sys_pipes_forever)
    ip.events.unregister('post_execute', stop_sys_pipes)
<|endoftext|>


<commit_before># Copyright 2017, Google LLC All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import logging
import threading
import uuid

import six

__all__ = (
    'HelperThreadRegistry',
    'QueueCallbackThread',
    'STOP',
)

_LOGGER = logging.getLogger(__name__)

_HelperThread = collections.namedtuple(
    'HelperThreads',
    ['name', 'thread', 'queue'],
)

# Helper thread stop indicator. This could be a sentinel object or None,
# but the sentinel object's ID can change if the process is forked, and
# None has the possibility of a user accidentally killing the helper
# thread.
STOP = uuid.uuid4()

class HelperThreadRegistry(object):
    def __init__(self):
        self._helper_threads = {}

    def __contains__(self, needle):
        return needle in self._helper_threads

    def start(self, name, queue, target, *args, **kwargs):
        """Create and start a helper thread.

        Args:
            name (str): The name of the helper thread.
            queue (Queue): A concurrency-safe queue.
            target (Callable): The target of the thread.
            args: Additional args passed to the thread constructor.
            kwargs: Additional kwargs passed to the thread constructor.

        Returns:
            threading.Thread: The created thread.
        """
        # Create and start the helper thread.
        thread = threading.Thread(
            name='Thread-ConsumerHelper-{}'.format(name),
            target=target,
            *args,
            **kwargs
        )
        thread.daemon = True
        thread.start()

        # Keep track of the helper thread, so we are able to stop it.
        self._helper_threads[name] = _HelperThread(name, thread, queue)
        _LOGGER.debug('Started helper thread {}'.format(name))
        return thread

    def stop(self, name):
        """Stops a helper thread.

        Sends the stop message and blocks until the thread joins.

        Args:
            name (str): The name of the thread.
        """
        # Attempt to retrieve the thread; if it is gone already, no-op.
        helper_thread = self._helper_threads.get(name)
        if helper_thread is None:
            return

        # Join the thread if it is still alive.
        if helper_thread.thread.is_alive():
            _LOGGER.debug('Stopping helper thread {}'.format(name))
            helper_thread.queue.put(STOP)
            helper_thread.thread.join()

        # Remove the thread from our tracking.
        self._helper_threads.pop(name, None)

    def stop_all(self):
        """Stop all helper threads."""
        # This could be more efficient by sending the stop signal to all
        # threads before joining any of them.
        for name in list(six.iterkeys(self._helper_threads)):
            self.stop(name)

class QueueCallbackThread(object):
    """A helper thread that executes a callback for every item in
    the queue.
    """
    def __init__(self, queue, callback):
        self.queue = queue
        self._callback = callback

    def __call__(self):
        while True:
            item = self.queue.get()
            if item == STOP:
                break

            # Run the callback. If any exceptions occur, log them and
            # continue.
            try:
                self._callback(item)
            except Exception as exc:
                _LOGGER.error('{class_}: {message}'.format(
                    class_=exc.__class__.__name__,
                    message=str(exc),
                ))
<commit_msg>Removing unused "extra" arguments in Pub/Sub helper thread start. (#4483)<commit_after># Copyright 2017, Google LLC All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import collections
import logging
import threading
import uuid

import six

__all__ = (
    'HelperThreadRegistry',
    'QueueCallbackThread',
    'STOP',
)

_LOGGER = logging.getLogger(__name__)

_HelperThread = collections.namedtuple(
    'HelperThreads',
    ['name', 'thread', 'queue'],
)

# Helper thread stop indicator. This could be a sentinel object or None,
# but the sentinel object's ID can change if the process is forked, and
# None has the possibility of a user accidentally killing the helper
# thread.
STOP = uuid.uuid4()

class HelperThreadRegistry(object):
    def __init__(self):
        self._helper_threads = {}

    def __contains__(self, needle):
        return needle in self._helper_threads

    def start(self, name, queue, target):
        """Create and start a helper thread.

        Args:
            name (str): The name of the helper thread.
            queue (Queue): A concurrency-safe queue.
            target (Callable): The target of the thread.

        Returns:
            threading.Thread: The created thread.
        """
        # Create and start the helper thread.
        thread = threading.Thread(
            name='Thread-ConsumerHelper-{}'.format(name),
            target=target,
        )
        thread.daemon = True
        thread.start()

        # Keep track of the helper thread, so we are able to stop it.
        self._helper_threads[name] = _HelperThread(name, thread, queue)
        _LOGGER.debug('Started helper thread {}'.format(name))
        return thread

    def stop(self, name):
        """Stops a helper thread.

        Sends the stop message and blocks until the thread joins.

        Args:
            name (str): The name of the thread.
        """
        # Attempt to retrieve the thread; if it is gone already, no-op.
        helper_thread = self._helper_threads.get(name)
        if helper_thread is None:
            return

        # Join the thread if it is still alive.
        if helper_thread.thread.is_alive():
            _LOGGER.debug('Stopping helper thread {}'.format(name))
            helper_thread.queue.put(STOP)
            helper_thread.thread.join()

        # Remove the thread from our tracking.
        self._helper_threads.pop(name, None)

    def stop_all(self):
        """Stop all helper threads."""
        # This could be more efficient by sending the stop signal to all
        # threads before joining any of them.
        for name in list(six.iterkeys(self._helper_threads)):
            self.stop(name)

class QueueCallbackThread(object):
    """A helper thread that executes a callback for every item in
    the queue.
    """
    def __init__(self, queue, callback):
        self.queue = queue
        self._callback = callback

    def __call__(self):
        while True:
            item = self.queue.get()
            if item == STOP:
                break

            # Run the callback. If any exceptions occur, log them and
            # continue.
            try:
                self._callback(item)
            except Exception as exc:
                _LOGGER.error('{class_}: {message}'.format(
                    class_=exc.__class__.__name__,
                    message=str(exc),
                ))
<|endoftext|>


<commit_before># -*- encoding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2004-2008 TINY SPRL. (http://tiny.be) All Rights Reserved.
#
# $Id$
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import wizard
import tools
import pooler

view_form_end = """<?xml version="1.0"?>
<form string="Language file loaded.">
    <image name="gtk-dialog-info" colspan="2"/>
    <group colspan="2" col="4">
        <separator string="Installation done" colspan="4"/>
        <label align="0.0" string="The selected language has been successfully installed.\nYou must change the preferences of the user and open a new menu to view changes." colspan="4"/>
    </group>
</form>"""

view_form = """<?xml version="1.0"?>
<form string="System Upgrade">
    <image name="gtk-dialog-info" colspan="2"/>
    <group colspan="2" col="4">
    <separator string="System Upgrade" colspan="4"/>
        <label align="0.0" string="Choose a language to install:" colspan="4"/>
        <field name="lang" colspan="4"/>
        <label align="0.0" string="Note that this operation may take a few minutes." colspan="4"/>
    </group>
</form>"""

class wizard_lang_install(wizard.interface):
    def _lang_install(self, cr, uid, data, context):
        lang = data['form']['lang']
        if lang:
            modobj = pooler.get_pool(cr.dbname).get('ir.module.module')
            mids = modobj.search(cr, uid, [('state', '=', 'installed')])
            modobj.update_translations(cr, uid, mids, lang)
        return {}

    def _get_language(sel, cr, uid, context):
        return tools.scan_languages()

    fields_form = {
        'lang': {'string':'Language', 'type':'selection', 'selection':_get_language,
        },
    }

    states = {
        'init': {
            'actions': [],
            'result': {'type': 'form', 'arch': view_form, 'fields': fields_form,
                'state': [
                    ('end', 'Cancel', 'gtk-cancel'),
                    ('start', 'Start installation', 'gtk-ok', True)
                ]
            }
        },
        'start': {
            'actions': [_lang_install],
            'result': {'type': 'form', 'arch': view_form_end, 'fields': {},
                'state': [
                    ('end', 'Ok', 'gtk-ok', True)
                ]
            }
        },
    }
wizard_lang_install('module.lang.install')

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<commit_msg>improve_translation_view<commit_after># -*- encoding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2004-2008 TINY SPRL. (http://tiny.be) All Rights Reserved.
#
# $Id$
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsability of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# garantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import wizard
import tools
import pooler

view_form_end = """<?xml version="1.0"?>
<form string="Language file loaded.">
    <image name="gtk-dialog-info" colspan="2"/>
    <group colspan="2" col="4">
        <separator string="Installation done" colspan="4"/>
        <label align="0.0" string="The selected language has been successfully installed.\nYou must change the preferences of the user and open a new menu to view changes." colspan="4"/>
    </group>
</form>"""

view_form = """<?xml version="1.0"?>
<form string="System Upgrade">
    <image name="gtk-dialog-info" colspan="2"/>
    <group colspan="2" col="4">
    <separator string="System Upgrade" colspan="4"/>
        <label align="0.0" string="Choose a language to install:" colspan="4"/>
        <field name="lang" colspan="4" required="1"/>
        <label align="0.0" string="Note that this operation may take a few minutes." colspan="4"/>
    </group>
</form>"""

class wizard_lang_install(wizard.interface):
    def _lang_install(self, cr, uid, data, context):
        lang = data['form']['lang']
        if lang:
            modobj = pooler.get_pool(cr.dbname).get('ir.module.module')
            mids = modobj.search(cr, uid, [('state', '=', 'installed')])
            modobj.update_translations(cr, uid, mids, lang)
        return {}

    def _get_language(sel, cr, uid, context):
        return tools.scan_languages()

    fields_form = {
        'lang': {'string':'Language', 'type':'selection', 'selection':_get_language,
        },
    }

    states = {
        'init': {
            'actions': [],
            'result': {'type': 'form', 'arch': view_form, 'fields': fields_form,
                'state': [
                    ('end', 'Cancel', 'gtk-cancel'),
                    ('start', 'Start installation', 'gtk-ok', True)
                ]
            }
        },
        'start': {
            'actions': [_lang_install],
            'result': {'type': 'form', 'arch': view_form_end, 'fields': {},
                'state': [
                    ('end', 'Ok', 'gtk-ok', True)
                ]
            }
        },
    }
wizard_lang_install('module.lang.install')

# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:

<|endoftext|>


<commit_before>import urllib2
import datetime
from lxml import etree

from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.conf import settings

from football365.models import Call

class Command(BaseCommand):
    help = "Fetch feeds from Football365 and pass to handlers."

    pipeline = {
        'table': ('table_raw', 'xml2dom', 'table_structure'),
        'fixtures': ('fixtures_raw', 'xml2dom', 'fixtures_structure'),
        'results': ('results_raw', 'xml2dom', 'results_structure'),
    }

    @transaction.commit_on_success
    def handle(self, *args, **options):
        for call in Call.objects.all():
            data = None
            for handler in self.pipeline[call.call_type]:
                data = getattr(self, handler)(call, data)

    def _raw(self, service, dt, di=None, ci=None):
        """Common method"""
        url = "%s/%s?cl=%s&dt=%s&di=%s" % (
            settings.FOOTBALL365['url'],
            service,
            settings.FOOTBALL365['client_id'],
            dt,
            di or ci
        )
        result = ''
        try:
            f = urllib2.urlopen(url)
            result = f.read()
            f.close()
        except urllib2.URLError:
            pass

        return result

    def xml2dom(self, call, data):
        return etree.fromstring(data)

    def table_raw(self, call, data):
        return self._raw('tablesfeed', 'TablesFS1', di=call.football365_service_id)

    def table_structure(self, call, data):
        result = []
        for row in data.findall('ROW'):
            result.append(dict(
                POSITION=int(row.get('POSITION')),
                TEAM=row[0].text,
                TEAMCODE=row[1].text,
                PLAYED=int(row[2].text),
                WON=int(row[3].text),
                DRAWN=int(row[4].text),
                LOST=int(row[5].text),
                GOALSFOR=int(row[6].text),
                GOALSAGAINST=int(row[7].text),
                GOALDIFFERENCE=int(row[8].text),
                POINTS=int(row[9].text),
            ))
        return result

    def fixtures_raw(self, call, data):
        return self._raw('fixturesfeed', 'Fixtures', di=call.football365_service_id)

    def fixtures_structure(self, call, data):
        result = []
        for day in data.findall('DAY'):
            for row in day[0][0][0].findall('MATCH'):

                # We want a UTC-0 time
                raw_starttime = '%s %s' % (day.get('DATE'), row[5].text)
                starttime = datetime.datetime.strptime(raw_starttime, '%d/%m/%Y %H:%M')
                tz = int(row[5].get('TIMEZONE').replace('GMT', ''))
                starttime = starttime - datetime.timedelta(hours=tz)

                result.append(dict(
                    HOMETEAM=row[0].text,
                    HOMETEAMCODE=row[1].text,
                    AWAYTEAM=row[2].text,
                    AWAYTEAMCODE=row[3].text,
                    VENUE=row[4].text,
                    STARTTIME=starttime
                ))
        return result

    def results_raw(self, call, data):
        return self._raw('resultsfeed', 'Results', di=call.football365_service_id)

    def results_structure(self, call, data):
        result = []
        for day in data.findall('DAY'):
            for row in day[0][0][0].findall('MATCH'):
                result.append(dict(
                    HOMETEAM=row[0].text,
                    HOMETEAMCODE=row[1].text,
                    AWAYTEAM=row[2].text,
                    AWAYTEAMCODE=row[3].text,
                    HOMETEAMSCORE=int(row[4].text),
                    AWAYTEAMSCORE=int(row[5].text),
                    DATE=datetime.datetime.strptime(day.get('DATE'), '%d/%m/%Y')
                ))
        return result
<commit_msg>Basic handling of live feeds<commit_after>import urllib2
import datetime
from lxml import etree

from django.core.management.base import BaseCommand, CommandError
from django.db import transaction
from django.conf import settings

from football365.models import Call

class Command(BaseCommand):
    help = "Fetch feeds from Football365 and pass to handlers."

    pipeline = {
        'table': ('table_raw', 'xml2dom', 'table_structure'),
        'fixtures': ('fixtures_raw', 'xml2dom', 'fixtures_structure'),
        'results': ('results_raw', 'xml2dom', 'results_structure'),
        'live': ('live_raw', 'xml2dom'),
    }

    @transaction.commit_on_success
    def handle(self, *args, **options):
        for call in Call.objects.all():
            data = None
            for handler in self.pipeline[call.call_type]:
                data = getattr(self, handler)(call, data)

    def _raw(self, service, dt=None, di=None, ci=None):
        """Common method"""
        url = "%s/%s?cl=%s" % (
            settings.FOOTBALL365['url'],
            service,
            settings.FOOTBALL365['client_id'],
        )
        if dt:
            url = url + "&dt=" + dt
        if di:
            url = url + "&di=" + str(di)
        if ci:
            url = url + "&ci=" + str(ci)

        result = ''
        try:
            f = urllib2.urlopen(url)
            result = f.read()
            f.close()
        except urllib2.URLError:
            pass

        return result

    def xml2dom(self, call, data):
        return etree.fromstring(data)

    def table_raw(self, call, data):
        return self._raw('tablesfeed', 'TablesFS1', di=call.football365_service_id)

    def table_structure(self, call, data):
        result = []
        for row in data.findall('ROW'):
            result.append(dict(
                POSITION=int(row.get('POSITION')),
                TEAM=row[0].text,
                TEAMCODE=row[1].text,
                PLAYED=int(row[2].text),
                WON=int(row[3].text),
                DRAWN=int(row[4].text),
                LOST=int(row[5].text),
                GOALSFOR=int(row[6].text),
                GOALSAGAINST=int(row[7].text),
                GOALDIFFERENCE=int(row[8].text),
                POINTS=int(row[9].text),
            ))
        return result

    def fixtures_raw(self, call, data):
        return self._raw('fixturesfeed', 'Fixtures', di=call.football365_service_id)

    def fixtures_structure(self, call, data):
        result = []
        for day in data.findall('DAY'):
            for row in day[0][0][0].findall('MATCH'):

                # We want a UTC-0 time
                raw_starttime = '%s %s' % (day.get('DATE'), row[5].text)
                starttime = datetime.datetime.strptime(raw_starttime, '%d/%m/%Y %H:%M')
                tz = int(row[5].get('TIMEZONE').replace('GMT', ''))
                starttime = starttime - datetime.timedelta(hours=tz)

                result.append(dict(
                    HOMETEAM=row[0].text,
                    HOMETEAMCODE=row[1].text,
                    AWAYTEAM=row[2].text,
                    AWAYTEAMCODE=row[3].text,
                    VENUE=row[4].text,
                    STARTTIME=starttime
                ))
        return result

    def results_raw(self, call, data):
        return self._raw('resultsfeed', 'Results', di=call.football365_service_id)

    def results_structure(self, call, data):
        result = []
        for day in data.findall('DAY'):
            for row in day[0][0][0].findall('MATCH'):
                result.append(dict(
                    HOMETEAM=row[0].text,
                    HOMETEAMCODE=row[1].text,
                    AWAYTEAM=row[2].text,
                    AWAYTEAMCODE=row[3].text,
                    HOMETEAMSCORE=int(row[4].text),
                    AWAYTEAMSCORE=int(row[5].text),
                    DATE=datetime.datetime.strptime(day.get('DATE'), '%d/%m/%Y')
                ))
        return result

    def live_raw(self, call, data):
        return self._raw('footballlive', ci=call.football365_service_id)

<|endoftext|>


<commit_before>"""
Models for two-body pairwise interactions
=========================================

.. currentmodule:: FFOMP.mmmodels.twobody

In this module, models for pairwise atomic interactions has been defined. And
they are all based on an abstract base class where most of the aspects of two-
body interactions are defined. The models includes,

.. autosummary::
    :toctree:

    Morse
    CubicSpline
    LeonardJones
    _TwoBodyI

"""

import abc
import itertools
import functools

from sympy import Float, Symbol, exp
from sympy.functions.special.bsplines import bspline_basis_set
import numpy as np
from numpy import linalg

from ..model import Model
from ..fitjob import ModelParam

#
# The abstract base class
# -----------------------
#

class _TwoBodyI(Model):

    """
    The abstract base class for two body interactions

    The subclasses of this class are required to define their own parameters
    and presentation methods. One more property ``energy_expr`` needs to be
    defined which gives a tuple whose first field is the expression of the
    interaction energy in terms of the models parameters and a distance symbol
    on the second field. Also the atomic symbols for the atomic types that the
    interaction applies to should be in the property ``atm_types``. Then the
    actual computation of static energy and atomic forces will be automatically
    carried out by the :py:meth:`__call__` method of this base class, based on
    the given energy profile expression. Note that the ``mols`` property in the
    data point are going to be used so that only interactions between atoms of
    different molecules will be included.

    """

    def __init__(self, force_match=True, energy_match=True):
        """Initializes a two-body interaction

        Currently it just sets if the forces are going to be modelled as well.

        :param bool force_match: If the forces are going to be matched.
        """

        self._force_match = force_match
        self._energy_match = energy_match

    @abc.abstractproperty
    def energy_expr(self):
        """The expression of the interaction energy

        It should be a tuple with the first field being the sympy expression of
        energy and the second field being the symbol that is used for the
        atomic distance in the expression.
        """

        pass

    @abc.abstractproperty
    def atm_types(self):
        """The types of the atoms that this interaction describes

        It needs to be a tuple of two strings, giving the symbols for the type
        of atoms whose interaction this model describes.
        """

        pass

    def __call__(self, data_pnt):
        """Calculates the static energy and atomic forces for the data point

        The staic energy and atomic forces will be computed.
        """

        # Before anything, make sure that we are operating on a valid data
        # point for potential energy surface scan fitting.
        try:
            symbs = data_pnt['atm_symbs']
            coords = data_pnt['atm_coords']
        except KeyError as exc:
            raise ValueError(
                'Property {} is required for two-body interaction '
                'models'.format(exc.args[0])
                )
        mols = data_pnt.get(
            'mols', [[i, ] for i, _ in enumerate(symbs)]
            )

        # First, get all the interacting pairs.
        pairs = _get_interacting_pairs(symbs, mols, self.atm_types)

        # Allocate the results and gradually add to them for each interacting
        # pair.
        if self._energy_match:
            energy = Float(0)
        if self._force_match:
            forces = [
                list(itertools.repeat(Float(0), 3))
                for _ in symbs
                ]

        # Lambdify the expression for energy and forces.
        energy_expr, dist_symb = self.energy_expr

        def energy_func(dist):
            """The energy function"""
            return energy_expr.subs(dist_symb, dist)

        if self._force_match:

            # Skip the expensive symbolic differentiation if force is not
            # going to be used.

            force_mag_expr = energy_expr.diff(dist_symb) * -1

            def force_mag_func(dist):
                """The force magnitude function

                In the direction from the interacting atom to the current atom.
                """
                return force_mag_expr.subs(dist_symb, dist)

        # Iterate over all the interacting pairs.
        for atm1, atm2 in pairs:

            # Calculate the distance and normalized vector from atom 1 to atom
            # 2.
            dist, vec = _get_dist_vec(coords[atm1], coords[atm2])

            # Test the cut-off.
            if hasattr(self, 'cut_off') and dist > self.cut_off:
                continue

            # Add the contribution to energy.
            if self._energy_match:
                energy += energy_func(dist)

            if self._force_match:
                # Compute the magnitude of the force.
                force_mag = force_mag_func(dist)

                # Add the force for the two atoms.
                #
                # Since the vector is from atom 1 to atom 2, we can treat atom
                # 2 first.
                for i in [atm2, atm1]:
                    for j, k in enumerate(vec):
                        forces[i][j] += k * force_mag
                        # Continue to the next axis
                        continue
                    # Flip the vector.
                    vec *= -1
                    # Continue to the next atom.
                    continue

            # Continue to the next atomic pair.
            continue

        # Return the result.
        res = {}
        if self._energy_match:
            res['static_energy'] = energy
        if self._force_match:
            res['atm_forces'] = forces
        return res

def _get_interacting_pairs(symbs, mols, symbs4inter):
    """Gets the list of interacting pairs on the system

    The pairs of indices of atomic in the symbol list will be returned. The
    interacting pairs must have the symbols required by the interaction and
    belongs to different molecules.

    :param symbs: The sequence of atomic symbols in the system.
    :param mols: A sequence of sequences of atomic indices, with each
        subsequence gives the indices of atomic of one molecule.
    :param symbs4inter: The pair of atomic symbols for the interaction.
    :returns: The list of atomic indices pairs of the interacting pairs of
        atoms for this interaction.
    :rtype: list
    """

    symbs4inter = tuple(sorted(symbs4inter))

    pairs = []

    for mol1, mol2 in itertools.combinations(mols, 2):
        for atms in itertools.product(mol1, mol2):
            if tuple(sorted(symbs[i] for i in atms)) == symbs4inter:
                pairs.append(atms)
            else:
                continue

    return pairs

def _get_dist_vec(pnt1, pnt2):
    """Gets the distance and normalized vector between two points

    The normalized vector will be an numpy vector from point one to point two.
    The two points do not need to be numpy arrays before hand.

    :param pnt1: The first point, as a iterable of coordinate values.
    :param pnt2: The second point.
    :returns: The Euclidean distance between the two points and the normalized
        vector from point one to point two.
    :rtype: tuple:
    """

    diff = np.array(
        [j - i for i, j in zip(pnt1, pnt2)],
        dtype=np.float
        )

    dist = linalg.norm(diff)

    return dist, diff / dist

#
# Potentials
# ----------
#

class Morse(_TwoBodyI):

    """The Morse interaction potential

    In this three parameter model, the interaction potential reads

    .. math::
        D_e \\left((1 - e^{-a(r - r_e)})^2 - 1\\right)

    """

    def __init__(self, atm_symbs, params, force_match=True):
        """Initializes a Morse interaction model

        :param atm_symbs: A pair of strings for the atomic symbols of the atoms
            whose interaction this model describes.
        :param params: The triple of initial guess, lower bound, and upper
            bound for the parameters, in the order of :math:`D_e`, :math:`a`,
            :math:`r_0`.
        """

        super().__init__(force_match=force_match)

        self._atm_types = tuple(str(i) for i in atm_symbs)
        if len(self._atm_types) != 2:
            raise ValueError(
                'Invalid number of atomic symbols!'
                )

        param_name_bases = [
            'DE', 'a', 'r0'
            ]
        self._model_params = []
        for name_base, spec in zip(param_name_bases, params):
            try:
                self._model_params.append(ModelParam(
                    symb=Symbol('_'.join((name_base, ) + self._atm_types)),
                    lower=spec[1], upper=spec[2],
                    init_guess=spec[0],
                    ))
            except IndexError:
                raise ValueError(
                    'Invalid parameter specification {} for {}'.format(
                        spec, name_base
                        )
                    )

    @property
    def atm_types(self):
        """The atomic types of interactions"""
        return self._atm_types

    @property
    def energy_expr(self):
        """The energy expression"""

        r = Symbol('r')
        de, a, r0 = [i.symb for i in self._model_params]

        return (
            de * ((exp(a * (r0 - r)) - 1) ** 2 - 1),
            r
            )

    @property
    def model_params(self):
        """The model parameters"""
        return self._model_params

    def present(self, param_vals, output):
        """Presents the results

        The results and a restart literal for the current object will be
        written.

        :param param_vals: A sequence for the values of the parameters.
        :param output: The output stream.
        """

        prt = functools.partial(print, file=output)

        prt('\n\n')
        prt('Morse parameters for {0[0]} and {0[1]}: De  a  r0'.format(
            self._atm_types
            ))
        prt('{0[0]!r}  {0[1]!r}  {0[2]!r}'.format(param_vals))
        prt('Restart literal:')
        prt('Morse({atm_types!r}, {params!r})'.format(
            atm_types=self._atm_types,
            params=[
                (j, i.lower, i.upper)
                for i, j in zip(self.model_params, param_vals)
                ]
            ))
        prt('\n\n')

        return None

class CubicSpline(_TwoBodyI):

    """
    Cubic spline two-body interaction model

    In this cubic spline model, the interaction energy is a linear
    superposition of cubic basis splines.
    """

    def __init__(self, atm_symbs, knots, force_match=True):
        """Initializes a cubic spline interaction model

        :param atm_symbs: A pair of strings for the atomic symbols of the atoms
            whose interaction this model describes.
        :param knots params: The knots in the cubic spline, given as a sequence
            of numbers for the locations. The last location will be taken as
            the cut-off for the interaction.
        """

        super().__init__(force_match=force_match)

        self._atm_types = tuple(str(i) for i in atm_symbs)
        if len(self._atm_types) != 2:
            raise ValueError(
                'Invalid number of atomic symbols!'
                )

        param_name_base = 'alpha'
        self._dist = Symbol('r')
        self._model_params = []
        self._basis = []
        self._knots = knots
        self.cut_off = max(knots)

        for idx, basis in enumerate(bspline_basis_set(3, knots, self._dist)):
            self._model_params.append(ModelParam(
                symb=Symbol(''.join([
                    param_name_base, self._atm_types[0], self.atm_types[1],
                    str(idx)
                    ])),
                lower=None, upper=None, init_guess=None
                ))
            self._basis.append(basis)
            continue

        self._energy_expr = sum(
            i * j.symb for i, j in zip(self._basis, self._model_params)
            )

    @property
    def atm_types(self):
        """The atomic types of interactions"""
        return self._atm_types

    @property
    def energy_expr(self):
        """The energy expression"""

        return (
            self._energy_expr, self._dist
            )

    @property
    def model_params(self):
        """The model parameters"""
        return self._model_params

    def present(self, param_vals, output):
        """Presents the results

        Very different from other models, this model will present itself as a
        list of energies for the knots points.

        :param param_vals: A sequence for the values of the parameters.
        :param output: The output stream.
        :returns: The list of the energy values at the knots points will be
            returned.
        :rtype: list
        """

        prt = functools.partial(print, file=output)

        prt('\n\n')
        prt('Cubic spline fitting results for {0[0]} and {0[1]}\n'.format(
            self._atm_types
            ))

        vals = []
        subs_dict = {
            i.symb: j
            for i, j in zip(self._model_params, param_vals)
            }

        for i in self._knots:
            subs_dict[self._dist] = i
            vals.append(
                float(self._energy_expr.subs(subs_dict).evalf())
                )
            prt('   {}     {}'.format(i, vals[-1]))
            continue
        prt('\n\n')

        return None
<commit_msg>Fix bug for disabling energy matching<commit_after>"""
Models for two-body pairwise interactions
=========================================

.. currentmodule:: FFOMP.mmmodels.twobody

In this module, models for pairwise atomic interactions has been defined. And
they are all based on an abstract base class where most of the aspects of two-
body interactions are defined. The models includes,

.. autosummary::
    :toctree:

    Morse
    CubicSpline
    LeonardJones
    _TwoBodyI

"""

import abc
import itertools
import functools

from sympy import Float, Symbol, exp
from sympy.functions.special.bsplines import bspline_basis_set
import numpy as np
from numpy import linalg

from ..model import Model
from ..fitjob import ModelParam

#
# The abstract base class
# -----------------------
#

class _TwoBodyI(Model):

    """
    The abstract base class for two body interactions

    The subclasses of this class are required to define their own parameters
    and presentation methods. One more property ``energy_expr`` needs to be
    defined which gives a tuple whose first field is the expression of the
    interaction energy in terms of the models parameters and a distance symbol
    on the second field. Also the atomic symbols for the atomic types that the
    interaction applies to should be in the property ``atm_types``. Then the
    actual computation of static energy and atomic forces will be automatically
    carried out by the :py:meth:`__call__` method of this base class, based on
    the given energy profile expression. Note that the ``mols`` property in the
    data point are going to be used so that only interactions between atoms of
    different molecules will be included.

    """

    def __init__(self, force_match=True, energy_match=True):
        """Initializes a two-body interaction

        Currently it just sets if the forces are going to be modelled as well.

        :param bool force_match: If the forces are going to be matched.
        """

        self._force_match = force_match
        self._energy_match = energy_match

    @abc.abstractproperty
    def energy_expr(self):
        """The expression of the interaction energy

        It should be a tuple with the first field being the sympy expression of
        energy and the second field being the symbol that is used for the
        atomic distance in the expression.
        """

        pass

    @abc.abstractproperty
    def atm_types(self):
        """The types of the atoms that this interaction describes

        It needs to be a tuple of two strings, giving the symbols for the type
        of atoms whose interaction this model describes.
        """

        pass

    def __call__(self, data_pnt):
        """Calculates the static energy and atomic forces for the data point

        The staic energy and atomic forces will be computed.
        """

        # Before anything, make sure that we are operating on a valid data
        # point for potential energy surface scan fitting.
        try:
            symbs = data_pnt['atm_symbs']
            coords = data_pnt['atm_coords']
        except KeyError as exc:
            raise ValueError(
                'Property {} is required for two-body interaction '
                'models'.format(exc.args[0])
                )
        mols = data_pnt.get(
            'mols', [[i, ] for i, _ in enumerate(symbs)]
            )

        # First, get all the interacting pairs.
        pairs = _get_interacting_pairs(symbs, mols, self.atm_types)

        # Allocate the results and gradually add to them for each interacting
        # pair.
        if self._energy_match:
            energy = Float(0)
        if self._force_match:
            forces = [
                list(itertools.repeat(Float(0), 3))
                for _ in symbs
                ]

        # Lambdify the expression for energy and forces.
        energy_expr, dist_symb = self.energy_expr

        def energy_func(dist):
            """The energy function"""
            return energy_expr.subs(dist_symb, dist)

        if self._force_match:

            # Skip the expensive symbolic differentiation if force is not
            # going to be used.

            force_mag_expr = energy_expr.diff(dist_symb) * -1

            def force_mag_func(dist):
                """The force magnitude function

                In the direction from the interacting atom to the current atom.
                """
                return force_mag_expr.subs(dist_symb, dist)

        # Iterate over all the interacting pairs.
        for atm1, atm2 in pairs:

            # Calculate the distance and normalized vector from atom 1 to atom
            # 2.
            dist, vec = _get_dist_vec(coords[atm1], coords[atm2])

            # Test the cut-off.
            if hasattr(self, 'cut_off') and dist > self.cut_off:
                continue

            # Add the contribution to energy.
            if self._energy_match:
                energy += energy_func(dist)

            if self._force_match:
                # Compute the magnitude of the force.
                force_mag = force_mag_func(dist)

                # Add the force for the two atoms.
                #
                # Since the vector is from atom 1 to atom 2, we can treat atom
                # 2 first.
                for i in [atm2, atm1]:
                    for j, k in enumerate(vec):
                        forces[i][j] += k * force_mag
                        # Continue to the next axis
                        continue
                    # Flip the vector.
                    vec *= -1
                    # Continue to the next atom.
                    continue

            # Continue to the next atomic pair.
            continue

        # Return the result.
        res = {}
        if self._energy_match:
            res['static_energy'] = energy
        if self._force_match:
            res['atm_forces'] = forces
        return res

def _get_interacting_pairs(symbs, mols, symbs4inter):
    """Gets the list of interacting pairs on the system

    The pairs of indices of atomic in the symbol list will be returned. The
    interacting pairs must have the symbols required by the interaction and
    belongs to different molecules.

    :param symbs: The sequence of atomic symbols in the system.
    :param mols: A sequence of sequences of atomic indices, with each
        subsequence gives the indices of atomic of one molecule.
    :param symbs4inter: The pair of atomic symbols for the interaction.
    :returns: The list of atomic indices pairs of the interacting pairs of
        atoms for this interaction.
    :rtype: list
    """

    symbs4inter = tuple(sorted(symbs4inter))

    pairs = []

    for mol1, mol2 in itertools.combinations(mols, 2):
        for atms in itertools.product(mol1, mol2):
            if tuple(sorted(symbs[i] for i in atms)) == symbs4inter:
                pairs.append(atms)
            else:
                continue

    return pairs

def _get_dist_vec(pnt1, pnt2):
    """Gets the distance and normalized vector between two points

    The normalized vector will be an numpy vector from point one to point two.
    The two points do not need to be numpy arrays before hand.

    :param pnt1: The first point, as a iterable of coordinate values.
    :param pnt2: The second point.
    :returns: The Euclidean distance between the two points and the normalized
        vector from point one to point two.
    :rtype: tuple:
    """

    diff = np.array(
        [j - i for i, j in zip(pnt1, pnt2)],
        dtype=np.float
        )

    dist = linalg.norm(diff)

    return dist, diff / dist

#
# Potentials
# ----------
#

class Morse(_TwoBodyI):

    """The Morse interaction potential

    In this three parameter model, the interaction potential reads

    .. math::
        D_e \\left((1 - e^{-a(r - r_e)})^2 - 1\\right)

    """

    def __init__(self, atm_symbs, params, force_match=True, energy_match=True):
        """Initializes a Morse interaction model

        :param atm_symbs: A pair of strings for the atomic symbols of the atoms
            whose interaction this model describes.
        :param params: The triple of initial guess, lower bound, and upper
            bound for the parameters, in the order of :math:`D_e`, :math:`a`,
            :math:`r_0`.
        """

        super().__init__(force_match=force_match, energy_match=energy_match)

        self._atm_types = tuple(str(i) for i in atm_symbs)
        if len(self._atm_types) != 2:
            raise ValueError(
                'Invalid number of atomic symbols!'
                )

        param_name_bases = [
            'DE', 'a', 'r0'
            ]
        self._model_params = []
        for name_base, spec in zip(param_name_bases, params):
            try:
                self._model_params.append(ModelParam(
                    symb=Symbol('_'.join((name_base, ) + self._atm_types)),
                    lower=spec[1], upper=spec[2],
                    init_guess=spec[0],
                    ))
            except IndexError:
                raise ValueError(
                    'Invalid parameter specification {} for {}'.format(
                        spec, name_base
                        )
                    )

    @property
    def atm_types(self):
        """The atomic types of interactions"""
        return self._atm_types

    @property
    def energy_expr(self):
        """The energy expression"""

        r = Symbol('r')
        de, a, r0 = [i.symb for i in self._model_params]

        return (
            de * ((exp(a * (r0 - r)) - 1) ** 2 - 1),
            r
            )

    @property
    def model_params(self):
        """The model parameters"""
        return self._model_params

    def present(self, param_vals, output):
        """Presents the results

        The results and a restart literal for the current object will be
        written.

        :param param_vals: A sequence for the values of the parameters.
        :param output: The output stream.
        """

        prt = functools.partial(print, file=output)

        prt('\n\n')
        prt('Morse parameters for {0[0]} and {0[1]}: De  a  r0'.format(
            self._atm_types
            ))
        prt('{0[0]!r}  {0[1]!r}  {0[2]!r}'.format(param_vals))
        prt('Restart literal:')
        prt('Morse({atm_types!r}, {params!r})'.format(
            atm_types=self._atm_types,
            params=[
                (j, i.lower, i.upper)
                for i, j in zip(self.model_params, param_vals)
                ]
            ))
        prt('\n\n')

        return None

class CubicSpline(_TwoBodyI):

    """
    Cubic spline two-body interaction model

    In this cubic spline model, the interaction energy is a linear
    superposition of cubic basis splines.
    """

    def __init__(self, atm_symbs, knots, force_match=True, energy_match=True):
        """Initializes a cubic spline interaction model

        :param atm_symbs: A pair of strings for the atomic symbols of the atoms
            whose interaction this model describes.
        :param knots params: The knots in the cubic spline, given as a sequence
            of numbers for the locations. The last location will be taken as
            the cut-off for the interaction.
        """

        super().__init__(force_match=force_match, energy_match=energy_match)

        self._atm_types = tuple(str(i) for i in atm_symbs)
        if len(self._atm_types) != 2:
            raise ValueError(
                'Invalid number of atomic symbols!'
                )

        param_name_base = 'alpha'
        self._dist = Symbol('r')
        self._model_params = []
        self._basis = []
        self._knots = knots
        self.cut_off = max(knots)

        for idx, basis in enumerate(bspline_basis_set(3, knots, self._dist)):
            self._model_params.append(ModelParam(
                symb=Symbol(''.join([
                    param_name_base, self._atm_types[0], self.atm_types[1],
                    str(idx)
                    ])),
                lower=None, upper=None, init_guess=None
                ))
            self._basis.append(basis)
            continue

        self._energy_expr = sum(
            i * j.symb for i, j in zip(self._basis, self._model_params)
            )

    @property
    def atm_types(self):
        """The atomic types of interactions"""
        return self._atm_types

    @property
    def energy_expr(self):
        """The energy expression"""

        return (
            self._energy_expr, self._dist
            )

    @property
    def model_params(self):
        """The model parameters"""
        return self._model_params

    def present(self, param_vals, output):
        """Presents the results

        Very different from other models, this model will present itself as a
        list of energies for the knots points.

        :param param_vals: A sequence for the values of the parameters.
        :param output: The output stream.
        :returns: The list of the energy values at the knots points will be
            returned.
        :rtype: list
        """

        prt = functools.partial(print, file=output)

        prt('\n\n')
        prt('Cubic spline fitting results for {0[0]} and {0[1]}\n'.format(
            self._atm_types
            ))

        vals = []
        subs_dict = {
            i.symb: j
            for i, j in zip(self._model_params, param_vals)
            }

        for i in self._knots:
            subs_dict[self._dist] = i
            vals.append(
                float(self._energy_expr.subs(subs_dict).evalf())
                )
            prt('   {}     {}'.format(i, vals[-1]))
            continue
        prt('\n\n')

        return None
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# PyCi
#
# Copyright (c) 2009, The PyCi Project
# Authors: Wu Ke <ngu.kho@gmail.com>
#          Chen Xing <cxcxcxcx@gmail.com>
# URL: <http://code.google.com/p/pyci>
# For license information, see COPYING

"""
Brill tagger
"""

from collections import defaultdict
import entity

class BrillRuleTemplate(object):
    pass

class BrillRule(object):
    pass

class BrillTagger(object):
    """A brill tagger"""

    def __init__(self, tagset, initial_tagger, rules=[]):
        """Construct a brill tagger

        @type tagset: a TagSet instance
        @param tagset: the tag set we are using
        @type initial_tagger: a function which takes a unicode string
        and returns tuples of (character, tag)
        @param initial_tagger: the tagger that produces the coarse
        tagging, it should match the tagset given
        @type rules: orderd list of BrillRules
        @param rules: rules to apply after tagging with initial_tagger
        """
        self.tagset = tagset
        self.itag = initial_tagger
        self.rules = rules

    def tag(self, sent):
        """Tag a sentence

        @type sent: a unicode string
        @param sent: the sentence to be tagged
        @return: a list of (character, tag) tuple
        """
        # we need to ensure that res is a list
        res = [i for i in self.itag(sent)]
        # construct the tag-to-index mapping for res
        tag_to_index = defaultdict(set)
        for idx, (character, tag) in enumerate(res):
            tag_to_index[tag].add(idx)
        # apply rules on res
        for rule in self.rules:
            # find out all changes
            changes = [i for i in tag_to_index[rule.from_tag]
                       if rule.applies(res, i)]
            # change their tags to rule.to_tag
            for i in changes:
                res[i] = (res[i][0], rule.to_tag)
                tag_to_index[rule.from_tag].remove(i)
                tag_to_index[rule.to_tag].add(i)
        # done
        return res

    def train(self, train, rule_templates, max_rules, min_score):
        """Train with given rule templates

        @type train: a list of (character, tag) tuples
        @param train: the training corpus
        @type rule_templates: a list of BrillRuleTemplate
        @param rule_templates: the rule templates to be used
        @type max_rules: integer
        @param max_rules: maximum number of rules to be used
        @type min_score: integer
        @param min_score: mininum improvement score a rule should get
        """
        # `declaration', these are to be used in helper functions
        res = []
        tag_to_index = defaultdict(set)
        error_idx = defaultdict(set)
        correct_idx = defaultdict(set)

        # helper functions, classes
        class HelperRule(object):
            """A helper for rule. Saves information needed in
            find_possible_rules and find_best_rule.
            """

            def __init__(self, rule):
                self.rule = rule
                self.correct_iter = iter(correct_idx[rule.from_tag])
                self.score = 0
                # find changes for error_idx[rule.from_tag]
                self.changes = [i for i in error_idx[rule.from_tag]
                                if rule.applies(res, i)]
                # calculate our score for fixing those errors
                for i in self.changes:
                    if train[i][1] == rule.to_tag:
                        # we are right
                        self.score += 1
                    else:
                        # we are wrong but it does not hurt. at least
                        # we are not bringing new errors.
                        pass

        def find_possible_rules():
            score_to_rules = defaultdict(set)
            rule_to_helper = {} # one rule <=> one helper
            max_score = 0
            for wrong_tag in error_idx:
                for idx in error_idx[wrong_tag]:
                    for rule in generate_rules(idx):
                        if rule in rule_to_helper:
                            # already have that rule, skipping
                            continue
                        helper = HelperRule(rule)
                        if helper.score >= min_score:
                            score_to_rules[helper.score].add(rule)
                            rule_to_helper[rule] = helper
                        if helper.score >= max_score:
                            max_score = helper.score
            return score_to_rules, rule_to_helper, max_score

        def find_best_rule(score_to_rules, rule_to_helper, max_score):
            # we find the best rule in the following way:
            #
            # 1. for each rule helper, we store the rule's changes and
            # its score.
            #
            # 2. treat score_to_rules like below,
            #
            # scores      rules
            #   x    -->  [...]
            #  x-1   -->  [...]
            #  x-2   -->  [...]
            #  ...   -->   ...
            #  min   -->  [...]
            #
            # 3. since we have `made' all changes to errors in
            # find_possible_rules, all the changes we are going to
            # make are on the correctly tagged characters -- that is
            # to say, we are not going to improve any score, only
            # lowering them.
            #
            # 4. so the algorithm goes like this:
            #
            # a. find a rule with currently maximum score, apply it to
            # any appliable correct indices.
            #
            # b. if the rule does not make any more changes to the
            # correct tags, it's the best rule, return it.
            #
            # c. otherwise, keep making mistakes and decreasing its
            # score till it no more has the highest score.
            #
            # d. repeat, someday we will reach stage b.
            while True:
                # take a max_score rule
                rule = score_to_rules[max_score].pop()
                # and its helper
                helper = rule_to_helper[rule]
                # just in case
                assert rule == helper.rule, \
                       "rule and the rule in its helper do not match"
                assert helper.score == max_score, \
                       "helper's score in wrong"
                # till it's no more the best
                while True:
                    # make the miserable mistake
                    try:
                        idx = helper.correct_iter.next()
                        if rule.applies(res, idx):
                            helper.changes.append(idx)
                            helper.score -= 1
                            score_to_rules[helper.score].add(rule)
                    except StopIteration:
                        return rule, score, helper
                # find the new max_score
                while len(score_to_rules[max_score]) == 0:
                    max_score -= 1
            assert 0, "You shouldn't be here"

        # tag raw sentence with our initial tagger, make sure res is a
        # list
        res = [i for i in self.itag(''.join([i[0] for i in train]))]
        # # construct the tag-to-index mapping for res
        # for idx, (character, tag) in enumerate(res):
        #     tag_to_index[tag].add(idx)
        # find out all error and correct indices
        for idx, (character, tag) in enumerate(res):
            if tag == train[idx][1]:
                correct_idx[tag].add(idx)
            else:
                error_idx[tag].add(idx)
        # new rule list
        rules = []
        while len(rules) <= max_rules:
            score_to_rules, rule_to_helper, max_score = find_possible_rules()
            if max_score >= min_score:
                rule, score, helper = find_best_rule(score_to_rules,
                                                     rule_to_helper,
                                                     max_score)
                if score >= min_score:
                    # commit changes
                    for i in helper.changes:
                        # update error_idx, correct_idx
                        if rule.from_tag != train[i][1]:
                            # we used to get it wrong, it might be
                            # still wrong. or when we're lucky, it's
                            # correct.
                            error_idx[res[i][1]].remove(i)
                            if rule.to_tag == train[i][1]:
                                # we got it correct!
                                correct_idx[rule.to_tag].add(i)
                            else:
                                # we got it wrong...
                                error_idx[rule.to_tag].add(i)
                        else:
                            # it was correct, now it's surely wrong
                            correct_idx[rule.from_tag].remove(i)
                            error_idx[rule.to_tag].add(i)
                        # finally, change the tag in res
                        res[i] = (res[i][0], new_rule.to_tag)
                    rules.append(new_rule)
                else:
                    break
            else:
                break
        # done
        return rules

def demo():
    from tagset.template import BETagSet
    be = BETagSet()

    class FakeRule:
        def __init__(self, from_tag, to_tag):
            self.from_tag = from_tag
            self.to_tag = to_tag
            self.applies = lambda x, y: True

    b_to_e = FakeRule('B', 'E')
    e_to_b = FakeRule('E', 'B')

    btag = BrillTagger(be, be.tag, [b_to_e])
    print btag.tag(['this', 'is', 'a', 'test'])

    btag = BrillTagger(be, be.tag, [b_to_e, e_to_b])
    print btag.tag(['this', 'is', 'a', 'test'])

if __name__ == "__main__":
    demo()
<commit_msg>Brill tagger(tested); Brill trainer(tested); Brill rules and templates undone<commit_after># -*- coding: utf-8 -*-

# PyCi
#
# Copyright (c) 2009, The PyCi Project
# Authors: Wu Ke <ngu.kho@gmail.com>
#          Chen Xing <cxcxcxcx@gmail.com>
# URL: <http://code.google.com/p/pyci>
# For license information, see COPYING

"""
Brill tagger
"""

from collections import defaultdict
from pprint import pprint

def my_print(*args):
    for n, i in enumerate(args):
        if len(str(i)) < 50:
            if n != len(args) - 1:
                print i,
            else:
                print i
        else:
            pprint(i)

import entity

class BrillRuleTemplate(object):
    pass

class BrillRule(object):
    pass

class BrillTagger(object):
    """A brill tagger"""

    def __init__(self, tagset, initial_tagger, rules=[], trace=0):
        """Construct a brill tagger

        @type tagset: a TagSet instance
        @param tagset: the tag set we are using
        @type initial_tagger: a function which takes a unicode string
        and returns tuples of (character, tag)
        @param initial_tagger: the tagger that produces the coarse
        tagging, it should match the tagset given
        @type rules: orderd list of BrillRules
        @param rules: rules to apply after tagging with initial_tagger
        """
        self.tagset = tagset
        self.itag = initial_tagger
        self.rules = rules
        self.trace = trace
        if trace:
            my_print("__init__:")
            my_print("\ttagset:", tagset)
            my_print("\titag:", initial_tagger)
            my_print("\trules:", rules)

    def tag(self, sent):
        """Tag a sentence

        @type sent: a unicode string
        @param sent: the sentence to be tagged
        @return: a list of (character, tag) tuple
        """
        # we need to ensure that res is a list
        res = [i for i in self.itag(sent)]
        if self.trace:
            my_print("tag:")
            my_print("\tinit:", res)
        # construct the tag-to-index mapping for res
        tag_to_index = defaultdict(set)
        for idx, (character, tag) in enumerate(res):
            tag_to_index[tag].add(idx)
        # apply rules on res
        for rule in self.rules:
            # find out all changes
            changes = [i for i in tag_to_index[rule.from_tag]
                       if rule.applies(res, i)]
            if self.trace:
                my_print("\tapply rule:", rule)
                my_print("\t\tchange:", changes)
            # change their tags to rule.to_tag
            for i in changes:
                res[i] = (res[i][0], rule.to_tag)
                tag_to_index[rule.from_tag].remove(i)
                tag_to_index[rule.to_tag].add(i)
        # done
        if self.trace:
            my_print("\tfinal:", res)
        return res

    def train(self, train, rule_templates, max_rules, min_score):
        """Train with given rule templates

        @type train: a list of (character, tag) tuples
        @param train: the training corpus
        @type rule_templates: a list of BrillRuleTemplate
        @param rule_templates: the rule templates to be used
        @type max_rules: integer
        @param max_rules: maximum number of rules to be used
        @type min_score: integer
        @param min_score: mininum improvement score a rule should get
        """
        # `declaration', these are to be used in helper functions
        trace = self.trace
        res = []
        tag_to_index = defaultdict(set)
        error_idx = defaultdict(set)
        correct_idx = defaultdict(set)

        if trace:
            my_print("train:")

        # helper functions, classes
        class HelperRule(object):
            """A helper for rule. Saves information needed in
            find_possible_rules and find_best_rule.
            """

            def __init__(self, rule):
                self.rule = rule
                self.correct_iter = iter(correct_idx[rule.from_tag])
                self.score = 0
                # find changes for error_idx[rule.from_tag]
                self.changes = [i for i in error_idx[rule.from_tag]
                                if rule.applies(res, i)]
                # calculate our score for fixing those errors
                for i in self.changes:
                    if train[i][1] == rule.to_tag:
                        # we are right
                        self.score += 1
                    else:
                        # we are wrong but it does not hurt. at least
                        # we are not bringing new errors.
                        pass
                if trace:
                    my_print("HelperRule@0x%08x" % id(self))
                    my_print("\trule:", self.rule)
                    my_print("\tscore:", self.score)
                    my_print("\tchanges:", self.changes)

        def generate_rules(idx):
            if trace:
                my_print("generate_rules(%d)" % idx)
            from_tag = res[idx][1]
            to_tag = train[idx][1]
            ans = [i.form_rule(from_tag, to_tag, res, idx)
                   for i in rule_templates]
            if trace:
                my_print("generate_rules:")
                my_print("\t", ans)
            return ans

        def find_possible_rules():
            score_to_rules = defaultdict(set)
            rule_to_helper = {} # one rule <=> one helper
            max_score = 0
            for wrong_tag in error_idx:
                for idx in error_idx[wrong_tag]:
                    for rule in generate_rules(idx):
                        if rule in rule_to_helper:
                            # already have that rule, skipping
                            continue
                        helper = HelperRule(rule)
                        if helper.score >= min_score:
                            score_to_rules[helper.score].add(rule)
                            rule_to_helper[rule] = helper
                        if helper.score >= max_score:
                            max_score = helper.score
            if trace:
                my_print("find_possible_rules:")
                my_print("\tmax_score:", max_score)
                my_print("\trules:", len(rule_to_helper))
                my_print("\tscore_to_rules", dict(score_to_rules))
            return score_to_rules, rule_to_helper, max_score

        def find_best_rule(score_to_rules, rule_to_helper, max_score):
            # we find the best rule in the following way:
            #
            # 1. for each rule helper, we store the rule's changes and
            # its score.
            #
            # 2. treat score_to_rules like below,
            #
            # scores      rules
            #   x    -->  [...]
            #  x-1   -->  [...]
            #  x-2   -->  [...]
            #  ...   -->   ...
            #  min   -->  [...]
            #
            # 3. since we have `made' all changes to errors in
            # find_possible_rules, all the changes we are going to
            # make are on the correctly tagged characters -- that is
            # to say, we are not going to improve any score, only
            # lowering them.
            #
            # 4. so the algorithm goes like this:
            #
            # a. find a rule with currently maximum score, apply it to
            # any appliable correct indices.
            #
            # b. if the rule does not make any more changes to the
            # correct tags, it's the best rule, return it.
            #
            # c. otherwise, keep making mistakes and decreasing its
            # score till it no more has the highest score.
            #
            # d. repeat, someday we will reach stage b.
            if trace:
                my_print("find_best_rule:")
            while max_score >= min_score:
                # take a max_score rule
                rule = score_to_rules[max_score].pop()
                if trace:
                    my_print("\trule:", rule)
                    my_print("\tmax_score:", max_score)
                # and its helper
                helper = rule_to_helper[rule]
                # just in case
                assert rule == helper.rule, \
                       "rule and the rule in its helper do not match"
                assert helper.score == max_score, \
                       "helper's score in wrong"
                # till it's no more the best
                while True:
                    # make the miserable mistake
                    try:
                        idx = helper.correct_iter.next()
                        if rule.applies(res, idx):
                            if trace:
                                my_print("\t", rule, "made a mistake @", idx)
                            helper.changes.append(idx)
                            helper.score -= 1
                            score_to_rules[helper.score].add(rule)
                            break
                    except StopIteration:
                        if trace:
                            my_print("\trule:", rule, "score:", helper.score)
                        return rule, helper.score, helper
                # find the new max_score
                while len(score_to_rules[max_score]) == 0:
                    max_score -= 1
            if trace:
                my_print("\tI am sorry...")
            return None, max_score, None

        # tag raw sentence with our initial tagger, make sure res is a
        # list
        res = [i for i in self.itag(''.join([i[0] for i in train]))]
        # # construct the tag-to-index mapping for res
        # for idx, (character, tag) in enumerate(res):
        #     tag_to_index[tag].add(idx)
        # find out all error and correct indices
        for idx, (character, tag) in enumerate(res):
            if tag == train[idx][1]:
                correct_idx[tag].add(idx)
            else:
                error_idx[tag].add(idx)
        if trace:
            my_print("train: ", [(res[i][0], train[i][1], res[i][1])
                              for i in range(len(res))])
            my_print("error:", dict(error_idx))
            my_print("correct:", dict(correct_idx))
        # new rule list
        rules = []
        while len(rules) <= max_rules:
            score_to_rules, rule_to_helper, max_score = find_possible_rules()
            if max_score >= min_score:
                rule, score, helper = find_best_rule(score_to_rules,
                                                     rule_to_helper,
                                                     max_score)
                if score >= min_score:
                    # commit changes
                    for i in helper.changes:
                        # update error_idx, correct_idx
                        if rule.from_tag != train[i][1]:
                            # we used to get it wrong, it might be
                            # still wrong. or when we're lucky, it's
                            # correct.
                            error_idx[res[i][1]].remove(i)
                            if rule.to_tag == train[i][1]:
                                # we got it correct!
                                correct_idx[rule.to_tag].add(i)
                            else:
                                # we got it wrong...
                                error_idx[rule.to_tag].add(i)
                        else:
                            # it was correct, now it's surely wrong
                            correct_idx[rule.from_tag].remove(i)
                            error_idx[rule.to_tag].add(i)
                        # finally, change the tag in res
                        res[i] = (res[i][0], rule.to_tag)
                    # add it to rules
                    rules.append(rule)
                    if trace:
                        my_print("train: ", [(res[i][0], train[i][1], res[i][1])
                                          for i in range(len(res))])
                        my_print("error:", dict(error_idx))
                        my_print("correct:", dict(correct_idx))
                else:
                    break
            else:
                break
        # done
        self.rules = rules

def demo():
    from tagset.template import BESTagSet
    bes = BESTagSet()

    class DummyRule:
        def __init__(self, from_tag, to_tag, test):
            self.from_tag = from_tag
            self.to_tag = to_tag
            self.test = test
        def __str__(self):
            return str([self.from_tag, self.to_tag, self.test])
        def __repr__(self):
            return repr([self.from_tag, self.to_tag, self.test])
        def __hash__(self):
            return hash(str(self))
        def __cmp__(self, other):
            if type(self) is type(other):
                return cmp(str(self), str(other))
            else:
                return 1
        def applies(self, res, idx):
            if res[idx][1] != self.from_tag:
                return False
            for pos, tag in self.test:
                if idx + pos >= 0 and idx + pos < len(res):
                    if res[idx + pos][1] != tag:
                        return False
            return True

    my_print("Test DummyRule")
    bb_to_sb = DummyRule('B', 'S', [(1, 'B')])
    se_to_sb = DummyRule('E', 'B', [(-1, 'S')])

    test = [('0', 'S'), ('1', 'E'), ('2', 'B'), ('3', 'B')]
    ans1 = [False, False, True, True]
    ans2 = [False, True, False, False]

    my_print(test)
    for i, (c, t) in enumerate(test):
        my_print(i, (c, t))

        assert bb_to_sb.applies(test,i ) == ans1[i]
        my_print("\tbb_to_sb:")
        if bb_to_sb.applies(test, i):
            my_print("yes")
        else:
            my_print("no")

        assert se_to_sb.applies(test, i) == ans2[i]
        my_print("\tse_to_sb:",)
        if se_to_sb.applies(test, i):
            my_print("yes")
        else:
            my_print("no")

    class DummyTemplate:
        def __init__(self, test_pos):
            self.test_pos = test_pos
        def form_rule(self, from_tag, to_tag, res, idx):
            assert res[idx][1] == from_tag
            test = []
            for i in self.test_pos:
                if idx + i >= 0 and idx + i < len(res):
                    test.append((i, res[idx + i][1]))
            return DummyRule(from_tag, to_tag, test)

    my_print("\nTest DummyTemplate")

    t1 = DummyTemplate([-1])
    t2 = DummyTemplate([1])

    r11 = t1.form_rule('S', 'B', test, 0)
    assert r11.test == []
    my_print(r11)

    r12 = t1.form_rule('E', 'B', test, 1)
    assert r12.test == [(-1, 'S')]
    my_print(r12)

    r21 = t2.form_rule('B', 'S', test, 2)
    assert r21.test == [(1, 'B')]
    my_print(r21)

    r22 = t2.form_rule('B', 'E', test, 3)
    assert r22.test == []
    my_print(r22)

    my_print("\nTest Brill trainer")

    templates = [t1, t2]
    train = [i for i in bes.tag(["This", "is", "a", "test", "."])]
    def stupid_tagger(sent):
        return [(i, "S") for i in sent]

    brill = BrillTagger(bes, stupid_tagger, trace=False)
    brill.train(train, templates, 20, 0)
    my_print(brill.rules)
    my_print(brill.tag("This is a test."))

if __name__ == "__main__":
    demo()
<|endoftext|>


<commit_before>"""
Solution to code eval interrupted bubble sort:
https://www.codeeval.com/open_challenges/158/
"""

def main(filename):
    """Open file and perform bubble_sort on each line."""
    with open(filename, 'r') as file_handle:
        for line in file_handle:
            interrupted_bs(line)

def interrupted_bs(line):
    """Parse input line and preform bubble sort X times.
    X is defined as the integer after '|' on line."""
    temp = line.split('|')
    num = int(temp[1])
    temp[0] = temp[0].rstrip()
    inputL = [int(x) for x in temp[0].split(' ')]
    for x in xrange(num):
        inputL = bubble_sort(inputL)
    # convert back to string
    print ' '.join(map(str, inputL))

def bubble_sort(inputL):
    """One iteration of bubble sort."""
    for num in xrange(len(inputL)-1):
        if inputL[num+1] < inputL[num]:
            inputL[num+1], inputL[num] = inputL[num], inputL[num+1]
    return inputL

if __name__ == '__main__':
    main('input.txt')
<commit_msg>use sys.argv<commit_after>"""
Solution to code eval interrupted bubble sort:
https://www.codeeval.com/open_challenges/158/
"""
import sys

def main(filename):
    """Open file and perform bubble_sort on each line."""
    with open(filename, 'r') as file_handle:
        for line in file_handle:
            interrupted_bs(line)

def interrupted_bs(line):
    """Parse input line and preform bubble sort X times.
    X is defined as the integer after '|' on line."""
    temp = line.split('|')
    num = int(temp[1])
    temp[0] = temp[0].rstrip()
    inputL = [int(x) for x in temp[0].split(' ')]
    for x in xrange(num):
        inputL = bubble_sort(inputL)
    # convert back to string
    print ' '.join(map(str, inputL))

def bubble_sort(inputL):
    """One iteration of bubble sort."""
    for num in xrange(len(inputL)-1):
        if inputL[num+1] < inputL[num]:
            inputL[num+1], inputL[num] = inputL[num], inputL[num+1]
    return inputL

if __name__ == '__main__':
    filename = sys.argv[1]
    main(filename)
<|endoftext|>