<commit_before>"""Event loop using a proactor and related classes.

A proactor is a "notify-on-completion" multiplexer.  Currently a
proactor is only implemented on Windows with IOCP.
"""

__all__ = ['BaseProactorEventLoop']

import socket

from . import base_events
from . import constants
from . import futures
from . import transports
from .log import logger

class _ProactorBasePipeTransport(transports.BaseTransport):
    """Base class for pipe and socket transports."""

    def __init__(self, loop, sock, protocol, waiter=None,
                 extra=None, server=None):
        super().__init__(extra)
        self._set_extra(sock)
        self._loop = loop
        self._sock = sock
        self._protocol = protocol
        self._server = server
        self._buffer = None  # None or bytearray.
        self._read_fut = None
        self._write_fut = None
        self._pending_write = 0
        self._conn_lost = 0
        self._closing = False  # Set when close() called.
        self._eof_written = False
        self._protocol_paused = False
        self.set_write_buffer_limits()
        if self._server is not None:
            self._server.attach(self)
        self._loop.call_soon(self._protocol.connection_made, self)
        if waiter is not None:
            self._loop.call_soon(waiter.set_result, None)

    def _set_extra(self, sock):
        self._extra['pipe'] = sock

    def close(self):
        if self._closing:
            return
        self._closing = True
        self._conn_lost += 1
        if not self._buffer and self._write_fut is None:
            self._loop.call_soon(self._call_connection_lost, None)
        if self._read_fut is not None:
            self._read_fut.cancel()

    def _fatal_error(self, exc):
        if not isinstance(exc, (BrokenPipeError, ConnectionResetError)):
            logger.exception('Fatal error for %s', self)
        self._force_close(exc)

    def _force_close(self, exc):
        if self._closing:
            return
        self._closing = True
        self._conn_lost += 1
        if self._write_fut:
            self._write_fut.cancel()
        if self._read_fut:
            self._read_fut.cancel()
        self._write_fut = self._read_fut = None
        self._pending_write = 0
        self._buffer = None
        self._loop.call_soon(self._call_connection_lost, exc)

    def _call_connection_lost(self, exc):
        try:
            self._protocol.connection_lost(exc)
        finally:
            # XXX If there is a pending overlapped read on the other
            # end then it may fail with ERROR_NETNAME_DELETED if we
            # just close our end.  First calling shutdown() seems to
            # cure it, but maybe using DisconnectEx() would be better.
            if hasattr(self._sock, 'shutdown'):
                self._sock.shutdown(socket.SHUT_RDWR)
            self._sock.close()
            server = self._server
            if server is not None:
                server.detach(self)
                self._server = None

    # XXX The next four methods are nearly identical to corresponding
    # ones in _SelectorTransport.  Maybe refactor buffer management to
    # share the implementations?  (Also these are really only needed
    # by _ProactorWritePipeTransport but since _buffer is defined on
    # the base class I am putting it here for now.)

    def _maybe_pause_protocol(self):
        size = self.get_write_buffer_size()
        if size <= self._high_water:
            return
        if not self._protocol_paused:
            self._protocol_paused = True
            try:
                self._protocol.pause_writing()
            except Exception:
                logger.exception('pause_writing() failed')

    def _maybe_resume_protocol(self):
        if (self._protocol_paused and
            self.get_write_buffer_size() <= self._low_water):
            self._protocol_paused = False
            try:
                self._protocol.resume_writing()
            except Exception:
                logger.exception('resume_writing() failed')

    def set_write_buffer_limits(self, high=None, low=None):
        if high is None:
            if low is None:
                high = 64*1024
            else:
                high = 4*low
        if low is None:
            low = high // 4
        if not high >= low >= 0:
            raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
                             (high, low))
        self._high_water = high
        self._low_water = low

    def get_write_buffer_size(self):
        size = self._pending_write
        if self._buffer is not None:
            size += len(self._buffer)
        return size

class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
                                 transports.ReadTransport):
    """Transport for read pipes."""

    def __init__(self, loop, sock, protocol, waiter=None,
                 extra=None, server=None):
        super().__init__(loop, sock, protocol, waiter, extra, server)
        self._read_fut = None
        self._paused = False
        self._loop.call_soon(self._loop_reading)

    def pause_reading(self):
        if self._closing:
            raise RuntimeError('Cannot pause_reading() when closing')
        if self._paused:
            raise RuntimeError('Already paused')
        self._paused = True

    def resume_reading(self):
        if not self._paused:
            raise RuntimeError('Not paused')
        self._paused = False
        if self._closing:
            return
        self._loop.call_soon(self._loop_reading, self._read_fut)

    def _loop_reading(self, fut=None):
        if self._paused:
            return
        data = None

        try:
            if fut is not None:
                assert self._read_fut is fut or (self._read_fut is None and
                                                 self._closing)
                self._read_fut = None
                data = fut.result()  # deliver data later in "finally" clause

            if self._closing:
                # since close() has been called we ignore any read data
                data = None
                return

            if data == b'':
                # we got end-of-file so no need to reschedule a new read
                return

            # reschedule a new read
            self._read_fut = self._loop._proactor.recv(self._sock, 4096)
        except ConnectionAbortedError as exc:
            if not self._closing:
                self._fatal_error(exc)
        except ConnectionResetError as exc:
            self._force_close(exc)
        except OSError as exc:
            self._fatal_error(exc)
        except futures.CancelledError:
            if not self._closing:
                raise
        else:
            self._read_fut.add_done_callback(self._loop_reading)
        finally:
            if data:
                self._protocol.data_received(data)
            elif data is not None:
                keep_open = self._protocol.eof_received()
                if not keep_open:
                    self.close()

class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
                                      transports.WriteTransport):
    """Transport for write pipes."""

    def write(self, data):
        if not isinstance(data, (bytes, bytearray, memoryview)):
            raise TypeError('data argument must be byte-ish (%r)',
                            type(data))
        if self._eof_written:
            raise RuntimeError('write_eof() already called')

        if not data:
            return

        if self._conn_lost:
            if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
                logger.warning('socket.send() raised exception.')
            self._conn_lost += 1
            return

        # Observable states:
        # 1. IDLE: _write_fut and _buffer both None
        # 2. WRITING: _write_fut set; _buffer None
        # 3. BACKED UP: _write_fut set; _buffer a bytearray
        # We always copy the data, so the caller can't modify it
        # while we're still waiting for the I/O to happen.
        if self._write_fut is None:  # IDLE -> WRITING
            assert self._buffer is None
            # Pass a copy, except if it's already immutable.
            self._loop_writing(data=bytes(data))
            # XXX Should we pause the protocol at this point
            # if len(data) > self._high_water?  (That would
            # require keeping track of the number of bytes passed
            # to a send() that hasn't finished yet.)
        elif not self._buffer:  # WRITING -> BACKED UP
            # Make a mutable copy which we can extend.
            self._buffer = bytearray(data)
            self._maybe_pause_protocol()
        else:  # BACKED UP
            # Append to buffer (also copies).
            self._buffer.extend(data)
            self._maybe_pause_protocol()

    def _loop_writing(self, f=None, data=None):
        try:
            assert f is self._write_fut
            self._write_fut = None
            self._pending_write = 0
            if f:
                f.result()
            if data is None:
                data = self._buffer
                self._buffer = None
            if not data:
                if self._closing:
                    self._loop.call_soon(self._call_connection_lost, None)
                if self._eof_written:
                    self._sock.shutdown(socket.SHUT_WR)
                # Now that we've reduced the buffer size, tell the
                # protocol to resume writing if it was paused.  Note that
                # we do this last since the callback is called immediately
                # and it may add more data to the buffer (even causing the
                # protocol to be paused again).
                self._maybe_resume_protocol()
            else:
                self._write_fut = self._loop._proactor.send(self._sock, data)
                if not self._write_fut.done():
                    assert self._pending_write == 0
                    self._pending_write = len(data)
                    self._write_fut.add_done_callback(self._loop_writing)
                    self._maybe_pause_protocol()
                else:
                    self._write_fut.add_done_callback(self._loop_writing)
        except ConnectionResetError as exc:
            self._force_close(exc)
        except OSError as exc:
            self._fatal_error(exc)

    def can_write_eof(self):
        return True

    def write_eof(self):
        self.close()

    def abort(self):
        self._force_close(None)

class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
    def __init__(self, *args, **kw):
        super().__init__(*args, **kw)
        self._read_fut = self._loop._proactor.recv(self._sock, 16)
        self._read_fut.add_done_callback(self._pipe_closed)

    def _pipe_closed(self, fut):
        if fut.cancelled():
            # the transport has been closed
            return
        assert fut is self._read_fut, (fut, self._read_fut)
        self._read_fut = None
        assert fut.result() == b''
        if self._write_fut is not None:
            self._force_close(exc)
        else:
            self.close()

class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
                                   _ProactorBaseWritePipeTransport,
                                   transports.Transport):
    """Transport for duplex pipes."""

    def can_write_eof(self):
        return False

    def write_eof(self):
        raise NotImplementedError

class _ProactorSocketTransport(_ProactorReadPipeTransport,
                               _ProactorBaseWritePipeTransport,
                               transports.Transport):
    """Transport for connected sockets."""

    def _set_extra(self, sock):
        self._extra['socket'] = sock
        try:
            self._extra['sockname'] = sock.getsockname()
        except (socket.error, AttributeError):
            pass
        if 'peername' not in self._extra:
            try:
                self._extra['peername'] = sock.getpeername()
            except (socket.error, AttributeError):
                pass

    def can_write_eof(self):
        return True

    def write_eof(self):
        if self._closing or self._eof_written:
            return
        self._eof_written = True
        if self._write_fut is None:
            self._sock.shutdown(socket.SHUT_WR)

class BaseProactorEventLoop(base_events.BaseEventLoop):

    def __init__(self, proactor):
        super().__init__()
        logger.debug('Using proactor: %s', proactor.__class__.__name__)
        self._proactor = proactor
        self._selector = proactor   # convenient alias
        self._self_reading_future = None
        self._accept_futures = {}   # socket file descriptor => Future
        self._granularity = max(proactor.resolution, self._granularity)
        proactor.set_loop(self)
        self._make_self_pipe()

    def _make_socket_transport(self, sock, protocol, waiter=None,
                               extra=None, server=None):
        return _ProactorSocketTransport(self, sock, protocol, waiter,
                                        extra, server)

    def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
                                    extra=None):
        return _ProactorDuplexPipeTransport(self,
                                            sock, protocol, waiter, extra)

    def _make_read_pipe_transport(self, sock, protocol, waiter=None,
                                  extra=None):
        return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra)

    def _make_write_pipe_transport(self, sock, protocol, waiter=None,
                                   extra=None):
        # We want connection_lost() to be called when other end closes
        return _ProactorWritePipeTransport(self,
                                           sock, protocol, waiter, extra)

    def close(self):
        if self._proactor is not None:
            self._close_self_pipe()
            self._proactor.close()
            self._proactor = None
            self._selector = None
            super().close()
        self._accept_futures.clear()

    def sock_recv(self, sock, n):
        return self._proactor.recv(sock, n)

    def sock_sendall(self, sock, data):
        return self._proactor.send(sock, data)

    def sock_connect(self, sock, address):
        return self._proactor.connect(sock, address)

    def sock_accept(self, sock):
        return self._proactor.accept(sock)

    def _socketpair(self):
        raise NotImplementedError

    def _close_self_pipe(self):
        if self._self_reading_future is not None:
            self._self_reading_future.cancel()
            self._self_reading_future = None
        self._ssock.close()
        self._ssock = None
        self._csock.close()
        self._csock = None
        self._internal_fds -= 1

    def _make_self_pipe(self):
        # A self-socket, really. :-)
        self._ssock, self._csock = self._socketpair()
        self._ssock.setblocking(False)
        self._csock.setblocking(False)
        self._internal_fds += 1
        self.call_soon(self._loop_self_reading)

    def _loop_self_reading(self, f=None):
        try:
            if f is not None:
                f.result()  # may raise
            f = self._proactor.recv(self._ssock, 4096)
        except:
            self.close()
            raise
        else:
            self._self_reading_future = f
            f.add_done_callback(self._loop_self_reading)

    def _write_to_self(self):
        self._csock.send(b'x')

    def _start_serving(self, protocol_factory, sock, ssl=None, server=None):
        if ssl:
            raise ValueError('IocpEventLoop is incompatible with SSL.')

        def loop(f=None):
            try:
                if f is not None:
                    conn, addr = f.result()
                    protocol = protocol_factory()
                    self._make_socket_transport(
                        conn, protocol,
                        extra={'peername': addr}, server=server)
                f = self._proactor.accept(sock)
            except OSError:
                if sock.fileno() != -1:
                    logger.exception('Accept failed')
                    sock.close()
            except futures.CancelledError:
                sock.close()
            else:
                self._accept_futures[sock.fileno()] = f
                f.add_done_callback(loop)

        self.call_soon(loop)

    def _process_events(self, event_list):
        pass    # XXX hard work currently done in poll

    def _stop_serving(self, sock):
        for future in self._accept_futures.values():
            future.cancel()
        self._proactor._stop_serving(sock)
        sock.close()
<commit_msg>Fix _ProactorWritePipeTransport._pipe_closed()<commit_after>"""Event loop using a proactor and related classes.

A proactor is a "notify-on-completion" multiplexer.  Currently a
proactor is only implemented on Windows with IOCP.
"""

__all__ = ['BaseProactorEventLoop']

import socket

from . import base_events
from . import constants
from . import futures
from . import transports
from .log import logger

class _ProactorBasePipeTransport(transports.BaseTransport):
    """Base class for pipe and socket transports."""

    def __init__(self, loop, sock, protocol, waiter=None,
                 extra=None, server=None):
        super().__init__(extra)
        self._set_extra(sock)
        self._loop = loop
        self._sock = sock
        self._protocol = protocol
        self._server = server
        self._buffer = None  # None or bytearray.
        self._read_fut = None
        self._write_fut = None
        self._pending_write = 0
        self._conn_lost = 0
        self._closing = False  # Set when close() called.
        self._eof_written = False
        self._protocol_paused = False
        self.set_write_buffer_limits()
        if self._server is not None:
            self._server.attach(self)
        self._loop.call_soon(self._protocol.connection_made, self)
        if waiter is not None:
            self._loop.call_soon(waiter.set_result, None)

    def _set_extra(self, sock):
        self._extra['pipe'] = sock

    def close(self):
        if self._closing:
            return
        self._closing = True
        self._conn_lost += 1
        if not self._buffer and self._write_fut is None:
            self._loop.call_soon(self._call_connection_lost, None)
        if self._read_fut is not None:
            self._read_fut.cancel()

    def _fatal_error(self, exc):
        if not isinstance(exc, (BrokenPipeError, ConnectionResetError)):
            logger.exception('Fatal error for %s', self)
        self._force_close(exc)

    def _force_close(self, exc):
        if self._closing:
            return
        self._closing = True
        self._conn_lost += 1
        if self._write_fut:
            self._write_fut.cancel()
        if self._read_fut:
            self._read_fut.cancel()
        self._write_fut = self._read_fut = None
        self._pending_write = 0
        self._buffer = None
        self._loop.call_soon(self._call_connection_lost, exc)

    def _call_connection_lost(self, exc):
        try:
            self._protocol.connection_lost(exc)
        finally:
            # XXX If there is a pending overlapped read on the other
            # end then it may fail with ERROR_NETNAME_DELETED if we
            # just close our end.  First calling shutdown() seems to
            # cure it, but maybe using DisconnectEx() would be better.
            if hasattr(self._sock, 'shutdown'):
                self._sock.shutdown(socket.SHUT_RDWR)
            self._sock.close()
            server = self._server
            if server is not None:
                server.detach(self)
                self._server = None

    # XXX The next four methods are nearly identical to corresponding
    # ones in _SelectorTransport.  Maybe refactor buffer management to
    # share the implementations?  (Also these are really only needed
    # by _ProactorWritePipeTransport but since _buffer is defined on
    # the base class I am putting it here for now.)

    def _maybe_pause_protocol(self):
        size = self.get_write_buffer_size()
        if size <= self._high_water:
            return
        if not self._protocol_paused:
            self._protocol_paused = True
            try:
                self._protocol.pause_writing()
            except Exception:
                logger.exception('pause_writing() failed')

    def _maybe_resume_protocol(self):
        if (self._protocol_paused and
            self.get_write_buffer_size() <= self._low_water):
            self._protocol_paused = False
            try:
                self._protocol.resume_writing()
            except Exception:
                logger.exception('resume_writing() failed')

    def set_write_buffer_limits(self, high=None, low=None):
        if high is None:
            if low is None:
                high = 64*1024
            else:
                high = 4*low
        if low is None:
            low = high // 4
        if not high >= low >= 0:
            raise ValueError('high (%r) must be >= low (%r) must be >= 0' %
                             (high, low))
        self._high_water = high
        self._low_water = low

    def get_write_buffer_size(self):
        size = self._pending_write
        if self._buffer is not None:
            size += len(self._buffer)
        return size

class _ProactorReadPipeTransport(_ProactorBasePipeTransport,
                                 transports.ReadTransport):
    """Transport for read pipes."""

    def __init__(self, loop, sock, protocol, waiter=None,
                 extra=None, server=None):
        super().__init__(loop, sock, protocol, waiter, extra, server)
        self._read_fut = None
        self._paused = False
        self._loop.call_soon(self._loop_reading)

    def pause_reading(self):
        if self._closing:
            raise RuntimeError('Cannot pause_reading() when closing')
        if self._paused:
            raise RuntimeError('Already paused')
        self._paused = True

    def resume_reading(self):
        if not self._paused:
            raise RuntimeError('Not paused')
        self._paused = False
        if self._closing:
            return
        self._loop.call_soon(self._loop_reading, self._read_fut)

    def _loop_reading(self, fut=None):
        if self._paused:
            return
        data = None

        try:
            if fut is not None:
                assert self._read_fut is fut or (self._read_fut is None and
                                                 self._closing)
                self._read_fut = None
                data = fut.result()  # deliver data later in "finally" clause

            if self._closing:
                # since close() has been called we ignore any read data
                data = None
                return

            if data == b'':
                # we got end-of-file so no need to reschedule a new read
                return

            # reschedule a new read
            self._read_fut = self._loop._proactor.recv(self._sock, 4096)
        except ConnectionAbortedError as exc:
            if not self._closing:
                self._fatal_error(exc)
        except ConnectionResetError as exc:
            self._force_close(exc)
        except OSError as exc:
            self._fatal_error(exc)
        except futures.CancelledError:
            if not self._closing:
                raise
        else:
            self._read_fut.add_done_callback(self._loop_reading)
        finally:
            if data:
                self._protocol.data_received(data)
            elif data is not None:
                keep_open = self._protocol.eof_received()
                if not keep_open:
                    self.close()

class _ProactorBaseWritePipeTransport(_ProactorBasePipeTransport,
                                      transports.WriteTransport):
    """Transport for write pipes."""

    def write(self, data):
        if not isinstance(data, (bytes, bytearray, memoryview)):
            raise TypeError('data argument must be byte-ish (%r)',
                            type(data))
        if self._eof_written:
            raise RuntimeError('write_eof() already called')

        if not data:
            return

        if self._conn_lost:
            if self._conn_lost >= constants.LOG_THRESHOLD_FOR_CONNLOST_WRITES:
                logger.warning('socket.send() raised exception.')
            self._conn_lost += 1
            return

        # Observable states:
        # 1. IDLE: _write_fut and _buffer both None
        # 2. WRITING: _write_fut set; _buffer None
        # 3. BACKED UP: _write_fut set; _buffer a bytearray
        # We always copy the data, so the caller can't modify it
        # while we're still waiting for the I/O to happen.
        if self._write_fut is None:  # IDLE -> WRITING
            assert self._buffer is None
            # Pass a copy, except if it's already immutable.
            self._loop_writing(data=bytes(data))
            # XXX Should we pause the protocol at this point
            # if len(data) > self._high_water?  (That would
            # require keeping track of the number of bytes passed
            # to a send() that hasn't finished yet.)
        elif not self._buffer:  # WRITING -> BACKED UP
            # Make a mutable copy which we can extend.
            self._buffer = bytearray(data)
            self._maybe_pause_protocol()
        else:  # BACKED UP
            # Append to buffer (also copies).
            self._buffer.extend(data)
            self._maybe_pause_protocol()

    def _loop_writing(self, f=None, data=None):
        try:
            assert f is self._write_fut
            self._write_fut = None
            self._pending_write = 0
            if f:
                f.result()
            if data is None:
                data = self._buffer
                self._buffer = None
            if not data:
                if self._closing:
                    self._loop.call_soon(self._call_connection_lost, None)
                if self._eof_written:
                    self._sock.shutdown(socket.SHUT_WR)
                # Now that we've reduced the buffer size, tell the
                # protocol to resume writing if it was paused.  Note that
                # we do this last since the callback is called immediately
                # and it may add more data to the buffer (even causing the
                # protocol to be paused again).
                self._maybe_resume_protocol()
            else:
                self._write_fut = self._loop._proactor.send(self._sock, data)
                if not self._write_fut.done():
                    assert self._pending_write == 0
                    self._pending_write = len(data)
                    self._write_fut.add_done_callback(self._loop_writing)
                    self._maybe_pause_protocol()
                else:
                    self._write_fut.add_done_callback(self._loop_writing)
        except ConnectionResetError as exc:
            self._force_close(exc)
        except OSError as exc:
            self._fatal_error(exc)

    def can_write_eof(self):
        return True

    def write_eof(self):
        self.close()

    def abort(self):
        self._force_close(None)

class _ProactorWritePipeTransport(_ProactorBaseWritePipeTransport):
    def __init__(self, *args, **kw):
        super().__init__(*args, **kw)
        self._read_fut = self._loop._proactor.recv(self._sock, 16)
        self._read_fut.add_done_callback(self._pipe_closed)

    def _pipe_closed(self, fut):
        if fut.cancelled():
            # the transport has been closed
            return
        assert fut.result() == b''
        if self._closing:
            assert self._read_fut is None
            return
        assert fut is self._read_fut, (fut, self._read_fut)
        self._read_fut = None
        if self._write_fut is not None:
            self._force_close(exc)
        else:
            self.close()

class _ProactorDuplexPipeTransport(_ProactorReadPipeTransport,
                                   _ProactorBaseWritePipeTransport,
                                   transports.Transport):
    """Transport for duplex pipes."""

    def can_write_eof(self):
        return False

    def write_eof(self):
        raise NotImplementedError

class _ProactorSocketTransport(_ProactorReadPipeTransport,
                               _ProactorBaseWritePipeTransport,
                               transports.Transport):
    """Transport for connected sockets."""

    def _set_extra(self, sock):
        self._extra['socket'] = sock
        try:
            self._extra['sockname'] = sock.getsockname()
        except (socket.error, AttributeError):
            pass
        if 'peername' not in self._extra:
            try:
                self._extra['peername'] = sock.getpeername()
            except (socket.error, AttributeError):
                pass

    def can_write_eof(self):
        return True

    def write_eof(self):
        if self._closing or self._eof_written:
            return
        self._eof_written = True
        if self._write_fut is None:
            self._sock.shutdown(socket.SHUT_WR)

class BaseProactorEventLoop(base_events.BaseEventLoop):

    def __init__(self, proactor):
        super().__init__()
        logger.debug('Using proactor: %s', proactor.__class__.__name__)
        self._proactor = proactor
        self._selector = proactor   # convenient alias
        self._self_reading_future = None
        self._accept_futures = {}   # socket file descriptor => Future
        self._granularity = max(proactor.resolution, self._granularity)
        proactor.set_loop(self)
        self._make_self_pipe()

    def _make_socket_transport(self, sock, protocol, waiter=None,
                               extra=None, server=None):
        return _ProactorSocketTransport(self, sock, protocol, waiter,
                                        extra, server)

    def _make_duplex_pipe_transport(self, sock, protocol, waiter=None,
                                    extra=None):
        return _ProactorDuplexPipeTransport(self,
                                            sock, protocol, waiter, extra)

    def _make_read_pipe_transport(self, sock, protocol, waiter=None,
                                  extra=None):
        return _ProactorReadPipeTransport(self, sock, protocol, waiter, extra)

    def _make_write_pipe_transport(self, sock, protocol, waiter=None,
                                   extra=None):
        # We want connection_lost() to be called when other end closes
        return _ProactorWritePipeTransport(self,
                                           sock, protocol, waiter, extra)

    def close(self):
        if self._proactor is not None:
            self._close_self_pipe()
            self._proactor.close()
            self._proactor = None
            self._selector = None
            super().close()
        self._accept_futures.clear()

    def sock_recv(self, sock, n):
        return self._proactor.recv(sock, n)

    def sock_sendall(self, sock, data):
        return self._proactor.send(sock, data)

    def sock_connect(self, sock, address):
        return self._proactor.connect(sock, address)

    def sock_accept(self, sock):
        return self._proactor.accept(sock)

    def _socketpair(self):
        raise NotImplementedError

    def _close_self_pipe(self):
        if self._self_reading_future is not None:
            self._self_reading_future.cancel()
            self._self_reading_future = None
        self._ssock.close()
        self._ssock = None
        self._csock.close()
        self._csock = None
        self._internal_fds -= 1

    def _make_self_pipe(self):
        # A self-socket, really. :-)
        self._ssock, self._csock = self._socketpair()
        self._ssock.setblocking(False)
        self._csock.setblocking(False)
        self._internal_fds += 1
        self.call_soon(self._loop_self_reading)

    def _loop_self_reading(self, f=None):
        try:
            if f is not None:
                f.result()  # may raise
            f = self._proactor.recv(self._ssock, 4096)
        except:
            self.close()
            raise
        else:
            self._self_reading_future = f
            f.add_done_callback(self._loop_self_reading)

    def _write_to_self(self):
        self._csock.send(b'x')

    def _start_serving(self, protocol_factory, sock, ssl=None, server=None):
        if ssl:
            raise ValueError('IocpEventLoop is incompatible with SSL.')

        def loop(f=None):
            try:
                if f is not None:
                    conn, addr = f.result()
                    protocol = protocol_factory()
                    self._make_socket_transport(
                        conn, protocol,
                        extra={'peername': addr}, server=server)
                f = self._proactor.accept(sock)
            except OSError:
                if sock.fileno() != -1:
                    logger.exception('Accept failed')
                    sock.close()
            except futures.CancelledError:
                sock.close()
            else:
                self._accept_futures[sock.fileno()] = f
                f.add_done_callback(loop)

        self.call_soon(loop)

    def _process_events(self, event_list):
        pass    # XXX hard work currently done in poll

    def _stop_serving(self, sock):
        for future in self._accept_futures.values():
            future.cancel()
        self._proactor._stop_serving(sock)
        sock.close()
<|endoftext|>


<commit_before>from django.db import connection
from laboratory.settings import TIME_ZONE
from utils.db import namedtuplefetchall

def get_patient_control_params(start_date, end_date, control_params, card_pk):
    with connection.cursor() as cursor:
        cursor.execute(
            """
            SELECT
                dp.patient_control_param_id, di.time_confirmation, value, dn.id as direction,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'DD.MM.YYYY') AS confirm,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'YYYY-MM') AS yearmonth_confirm
            FROM directions_paraclinicresult
            LEFT JOIN directory_paraclinicinputfield dp on directions_paraclinicresult.field_id = dp.id
            LEFT JOIN directions_issledovaniya di on directions_paraclinicresult.issledovaniye_id=di.id
            LEFT JOIN directions_napravleniya dn on dn.id=di.napravleniye_id
            WHERE
                dn.client_id = %(card_pk)s AND
                dp.patient_control_param_id in %(control_params)s AND
                di.time_confirmation AT TIME ZONE %(tz)s BETWEEN %(start_date)s AND %(end_date)s
            ORDER BY dp.patient_control_param_id, di.time_confirmation
            """,
            params={'start_date': start_date, 'end_date': end_date, 'control_params': control_params, 'card_pk': card_pk, 'tz': TIME_ZONE},
        )

        rows = namedtuplefetchall(cursor)
    return rows
<commit_msg>Laboratory control params<commit_after>from django.db import connection
from laboratory.settings import TIME_ZONE
from utils.db import namedtuplefetchall

def get_patient_control_params(start_date, end_date, control_params, card_pk):
    with connection.cursor() as cursor:
        cursor.execute(
            """
            SELECT
                dp.patient_control_param_id,
                di.time_confirmation,
                value,
                dn.id as direction,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'DD.MM.YYYY') AS confirm,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'YYYY-MM') AS yearmonth_confirm
            FROM directions_paraclinicresult
            LEFT JOIN directory_paraclinicinputfield dp on directions_paraclinicresult.field_id = dp.id
            LEFT JOIN directions_issledovaniya di on directions_paraclinicresult.issledovaniye_id=di.id
            LEFT JOIN directions_napravleniya dn on dn.id=di.napravleniye_id
            WHERE
                dn.client_id = %(card_pk)s AND
                dp.patient_control_param_id in %(control_params)s AND
                di.time_confirmation AT TIME ZONE %(tz)s BETWEEN %(start_date)s AND %(end_date)s
            UNION
            SELECT
                df.patient_control_param_id,
                di.time_confirmation,
                value,
                dn.id,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'DD.MM.YYYY') AS confirm,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'YYYY-MM') AS yearmonth_confirm
            FROM directions_result
            LEFT JOIN directory_fractions df on directions_result.fraction_id = df.id
            LEFT JOIN directions_issledovaniya di on directions_result.issledovaniye_id = di.id
            LEFT JOIN directions_napravleniya dn on dn.id=di.napravleniye_id
            WHERE
                dn.client_id = %(card_pk)s AND
                df.patient_control_param_id in %(control_params)s AND
                di.time_confirmation AT TIME ZONE %(tz)s BETWEEN %(start_date)s AND %(end_date)s
            ORDER BY patient_control_param_id, time_confirmation
            """,
            params={'start_date': start_date, 'end_date': end_date, 'control_params': control_params, 'card_pk': card_pk, 'tz': TIME_ZONE},
        )

        rows = namedtuplefetchall(cursor)
    return rows

def get_laboratory_patient_control_params(start_date, end_date, control_params, card_pk):
    with connection.cursor() as cursor:
        cursor.execute(
            """
            SELECT
                dp.patient_control_param_id, di.time_confirmation, value, dn.id as direction,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'DD.MM.YYYY') AS confirm,
                to_char(di.time_confirmation AT TIME ZONE %(tz)s, 'YYYY-MM') AS yearmonth_confirm
            FROM directions_result
            LEFT JOIN directory_fractions dp on directions_result.fraction_id = dp.id = dp.id
            LEFT JOIN directions_issledovaniya di on directions_result.issledovaniye_id = di.id
            LEFT JOIN directions_napravleniya dn on dn.id=di.napravleniye_id
            WHERE
                dn.client_id = %(card_pk)s AND
                dp.patient_control_param_id in %(control_params)s AND
                di.time_confirmation AT TIME ZONE %(tz)s BETWEEN %(start_date)s AND %(end_date)s
            ORDER BY dp.patient_control_param_id, di.time_confirmation
            """,
            params={'start_date': start_date, 'end_date': end_date, 'control_params': control_params, 'card_pk': card_pk, 'tz': TIME_ZONE},
        )

        rows = namedtuplefetchall(cursor)
    return rows
<|endoftext|>


<commit_before>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

import json
import argparse
import logging
import os
import sys
import pkg_resources
import locale

from atomic_reactor import set_logging
from atomic_reactor.api import (build_image_here, build_image_in_privileged_container,
                                build_image_using_hosts_docker)
from atomic_reactor.constants import CONTAINER_BUILD_JSON_PATH, DESCRIPTION, PROG
from atomic_reactor.buildimage import BuildImageBuilder
from atomic_reactor.inner import build_inside, BuildResults
from atomic_reactor.util import process_substitutions

logger = logging.getLogger('atomic_reactor')

def cli_create_build_image(args):
    b = BuildImageBuilder(reactor_tarball_path=args.reactor_tarball_path,
                          reactor_local_path=args.reactor_local_path,
                          reactor_remote_path=args.reactor_remote_git,
                          use_official_reactor_git=args.reactor_latest)
    try:
        b.create_image(args.dockerfile_dir_path, args.image, use_cache=args.use_cache)
    except RuntimeError:
        logger.error("build failed")
        sys.exit(1)
    sys.exit(0)

def cli_build_image(args):
    if args.plugin_files:
        args.plugin_files = [os.path.abspath(f) for f in args.plugin_files]
    if args.source__provider == 'json':
        with open(args.json_path) as json_fp:
            common_kwargs = json.load(json_fp)
        if args.substitute:
            process_substitutions(common_kwargs, args.substitute)
    else:
        common_kwargs = construct_kwargs(**vars(args))
    response = BuildResults()
    if args.method == "hostdocker":
        response = build_image_using_hosts_docker(args.build_image, **common_kwargs)
    elif args.method == "privileged":
        response = build_image_in_privileged_container(args.build_image, **common_kwargs)
    elif args.method == 'here':
        build_result = build_image_here(plugin_files=args.plugin_files, **common_kwargs)
        if build_result.is_failed():
            response.return_code = -1
        else:
            response.return_code = 0

    if response.return_code != 0:
        logger.error("build failed")
    sys.exit(response.return_code)

def construct_kwargs(**kwargs):
    ret = {}
    ret['source'] = {'provider_params': {}}

    # extend this when adding more args that should be passed to build_* functions
    recognized_kwargs = ['image', 'parent_registry', 'parent_registry_insecure',
                         'target_registries', 'target_registries_insecure',
                         'dont_pull_base_image']

    def is_recognized_kwarg(x):
        return x in recognized_kwargs or x.startswith('source__')

    for k, v in kwargs.items():
        if is_recognized_kwarg(k):
            if k.startswith('source__provider_params__'):
                ret['source']['provider_params'][k.split('__')[-1]] = v
            elif k.startswith('source__'):
                ret['source'][k.split('__')[-1]] = v
            else:
                ret[k] = v

    return ret

def cli_inside_build(args):
    build_inside(input_method=args.input, input_args=args.input_arg,
                 substitutions=args.substitute)

class CLI(object):
    def __init__(self, formatter_class=argparse.HelpFormatter, prog=PROG):
        self.parser = argparse.ArgumentParser(
            prog=prog,
            description=DESCRIPTION,
            formatter_class=formatter_class,
        )
        self.build_parser = None
        self.bi_parser = None
        self.ib_parser = None

        locale.setlocale(locale.LC_ALL, '')

    def set_arguments(self):
        try:
            version = pkg_resources.get_distribution("atomic_reactor").version
        except pkg_resources.DistributionNotFound:
            version = "GIT"

        exclusive_group = self.parser.add_mutually_exclusive_group()
        exclusive_group.add_argument("-q", "--quiet", action="store_true")
        exclusive_group.add_argument("-v", "--verbose", action="store_true")
        exclusive_group.add_argument("-V", "--version", action="version", version=version)

        subparsers = self.parser.add_subparsers(help='commands')

        # BUILDING IMAGES

        self.build_parser = subparsers.add_parser(
            'build',
            usage="%s [OPTIONS] build" % PROG,
            description='This command enables you to build images. '
                        'Currently, you can build images from git repo, local path or '
                        'a json file with full build specification. '
                        'There are several methods for performing the build: '
                        'inside a build container using docker from host, '
                        'inside a build container using new instance of docker, '
                        'or within current environment.'
        )

        def cli_build_with_source(args):
            if args.source__provider is None:
                self.build_parser.print_help()
            else:
                cli_build_image(args)

        self.build_parser.set_defaults(func=cli_build_with_source)
        self.generate_source_types_subparsers()

        # CREATE BUILD IMAGE

        self.bi_parser = subparsers.add_parser('create-build-image',
                                               usage="%s [OPTIONS] create-build-image" % PROG,
                                               description='Create build image; Atomic Reactor '
                                               'installs itself inside and is capable of '
                                               'building images within this image.')
        self.bi_parser.set_defaults(func=cli_create_build_image)
        reactor_source = self.bi_parser.add_mutually_exclusive_group()
        reactor_source.add_argument("--reactor-latest", action='store_true',
                                    help="put latest Atomic Reactor inside (from public git)")
        reactor_source.add_argument("--reactor-remote-git", action='store',
                                    help="URL to git repo with Atomic Reactor (has to contain "
                                    "setup.py)")
        reactor_source.add_argument("--reactor-local-path", action='store',
                                    help="path to directory with Atomic Reactor (has to contain "
                                    "setup.py)")
        reactor_source.add_argument("--reactor-tarball-path", action='store',
                                    help="path to distribution tarball with Atomic Reactor")
        self.bi_parser.add_argument("dockerfile_dir_path", action="store",
                                    metavar="DOCKERFILE_DIR_PATH",
                                    help="path to directory with Dockerfile")
        self.bi_parser.add_argument("image", action='store', metavar="IMAGE",
                                    help="name under the image will be accessible")
        self.bi_parser.add_argument("--use-cache", action='store_true', default=False,
                                    help="use cache to build image (may be faster, but not up "
                                    "to date)")

        # inside build
        self.ib_parser = subparsers.add_parser(
            'inside-build',
            usage="%s [OPTIONS] inside-build" % PROG,
            description="We do expect we are inside container, therefore we'll read "
                        "build configuration from json at '%s'" % CONTAINER_BUILD_JSON_PATH +
                        "and when the build is done, results are written in that dir so Atomic "
                        "Reactor from host may read those.")
        self.ib_parser.add_argument("--input", action='store', default="auto",
                                    help="input plugin name (determined automatically unless "
                                    "given)")
        self.ib_parser.add_argument("--input-arg", action='append',
                                    help="argument for input plugin (in form of 'key=value'), "
                                    "see input plugins to know what arguments they accept (can "
                                    "be specified multiple times)")
        self.ib_parser.add_argument("--dont-pull-base-image", action='store_true',
                                    help="don't pull or update base image specified in "
                                    "dockerfile")
        self.ib_parser.add_argument("--substitute", action='append',
                                    help="substitute values in build json (key=value, or "
                                         "plugin_type.plugin_name.key=value)")
        self.ib_parser.set_defaults(func=cli_inside_build)

    def generate_source_types_subparsers(self):
        build_subparsers = self.build_parser.add_subparsers(help='select source provider to use',
                                                            dest='source__provider')
        self.source_types_parsers = {
            'git': build_subparsers.add_parser(
                'git',
                help='build from a git repo',
                usage="%s [OPTIONS] build git" % PROG),
            'path': build_subparsers.add_parser(
                'path',
                help='build from a local path',
                usage="%s [OPTIONS] build path" % PROG),
        }
        source_types_uri_help = {
            'git': 'URI of the git repo',
            'path': 'URI of the directory with source (e.g. "file:///foo/bar") or\
                    path (e.g. "/foo/bar"); path can be relative',
        }
        source_types_df_path_help = {
            'git': 'path to Dockerfile within git repo (default is ./)',
            'path': 'path to Dockerfile within given directory (default is ./)',
        }

        # add arguments that are common to non-json source types
        for st, stp in self.source_types_parsers.items():
            stp.add_argument(
                "--image", action='store', required=True,
                help="name under which the image will be accessible")
            stp.add_argument(
                "--uri", action='store', metavar="URI", required=True,
                help=source_types_uri_help[st], dest='source__uri')
            stp.add_argument(
                "--df-path", action='store', help=source_types_df_path_help[st],
                dest='source__dockerfile_path')
            stp.add_argument(
                "--source-registry", action='store', metavar="REGISTRY",
                help="registry to pull base image from")
            stp.add_argument(
                "--source-registry-insecure", action='store_true',
                help="allow connecting to source registry over plain http")
            stp.add_argument(
                "--target-registries", action='store', nargs="*", metavar="REGISTRY",
                help="list of registries to push image to")
            stp.add_argument(
                "--target-registries-insecure", action='store_true',
                help="allow connecting to target registries over plain http")
            stp.add_argument(
                "--dont-pull-base-image", action='store_true',
                help="don't pull or update base image specified in dockerfile")

        # add the "json" subparser that allows to build from a json build file
        self.source_types_parsers['json'] = build_subparsers.add_parser(
            'json',
            help='load build configuration from json file',
            usage="%s [OPTIONS] build json" % PROG)

        # add arguments common to both non-json and json source types
        for stp in self.source_types_parsers.values():
            stp.add_argument(
                "--build-image", action='store',
                help="name of build image to use (build image type has to match method)")
            stp.add_argument(
                "--load-plugin", action="store", nargs="*", metavar="PLUGIN_FILE",
                dest="plugin_files", help="list of files where plugins live")
            stp.add_argument(
                "--method", action='store', required=True,
                choices=["hostdocker", "privileged", "here"],
                help="choose method for building image: "
                     "'hostdocker' mounts socket inside privileged container to use docker from "
                     "host, 'privileged' runs separate docker instance inside privileged "
                     "container and 'here' executes build in current environment")

        # add sourcetype-specific arguments now
        self.source_types_parsers['git'].add_argument(
            '--commit', action='store',
            dest='source__provider_params__git_commit',
            help="checkout this commit (default is master)")
        self.source_types_parsers['json'].add_argument(
            'json_path', metavar='JSON_PATH',
            help='path to the build json')
        self.source_types_parsers['json'].add_argument(
            '--substitute', nargs='*', metavar='SUBSTITUTE',
            help='provide substitutions for json in form "foo.bar=spam"')

    def run(self):
        self.set_arguments()
        args = self.parser.parse_args()
        logging.captureWarnings(True)

        if args.verbose:
            set_logging(level=logging.DEBUG)
        elif args.quiet:
            set_logging(level=logging.WARNING)
        else:
            set_logging(level=logging.INFO)
        try:
            args.func(args)
        except AttributeError:
            if hasattr(args, 'func'):
                raise
            else:
                self.parser.print_help()
        except KeyboardInterrupt:
            pass
        except Exception as ex:
            if args.verbose:
                raise
            else:
                logger.error("exception caught: %r", ex)

def run():
    cli = CLI()
    cli.run()

if __name__ == '__main__':
    run()
<commit_msg>Update help for inside-build CLI command<commit_after>"""
Copyright (c) 2015 Red Hat, Inc
All rights reserved.

This software may be modified and distributed under the terms
of the BSD license. See the LICENSE file for details.
"""

import json
import argparse
import logging
import os
import sys
import pkg_resources
import locale

from atomic_reactor import set_logging
from atomic_reactor.api import (build_image_here, build_image_in_privileged_container,
                                build_image_using_hosts_docker)
from atomic_reactor.constants import CONTAINER_BUILD_JSON_PATH, DESCRIPTION, PROG
from atomic_reactor.buildimage import BuildImageBuilder
from atomic_reactor.inner import build_inside, BuildResults
from atomic_reactor.util import process_substitutions

logger = logging.getLogger('atomic_reactor')

def cli_create_build_image(args):
    b = BuildImageBuilder(reactor_tarball_path=args.reactor_tarball_path,
                          reactor_local_path=args.reactor_local_path,
                          reactor_remote_path=args.reactor_remote_git,
                          use_official_reactor_git=args.reactor_latest)
    try:
        b.create_image(args.dockerfile_dir_path, args.image, use_cache=args.use_cache)
    except RuntimeError:
        logger.error("build failed")
        sys.exit(1)
    sys.exit(0)

def cli_build_image(args):
    if args.plugin_files:
        args.plugin_files = [os.path.abspath(f) for f in args.plugin_files]
    if args.source__provider == 'json':
        with open(args.json_path) as json_fp:
            common_kwargs = json.load(json_fp)
        if args.substitute:
            process_substitutions(common_kwargs, args.substitute)
    else:
        common_kwargs = construct_kwargs(**vars(args))
    response = BuildResults()
    if args.method == "hostdocker":
        response = build_image_using_hosts_docker(args.build_image, **common_kwargs)
    elif args.method == "privileged":
        response = build_image_in_privileged_container(args.build_image, **common_kwargs)
    elif args.method == 'here':
        build_result = build_image_here(plugin_files=args.plugin_files, **common_kwargs)
        if build_result.is_failed():
            response.return_code = -1
        else:
            response.return_code = 0

    if response.return_code != 0:
        logger.error("build failed")
    sys.exit(response.return_code)

def construct_kwargs(**kwargs):
    ret = {}
    ret['source'] = {'provider_params': {}}

    # extend this when adding more args that should be passed to build_* functions
    recognized_kwargs = ['image', 'parent_registry', 'parent_registry_insecure',
                         'target_registries', 'target_registries_insecure',
                         'dont_pull_base_image']

    def is_recognized_kwarg(x):
        return x in recognized_kwargs or x.startswith('source__')

    for k, v in kwargs.items():
        if is_recognized_kwarg(k):
            if k.startswith('source__provider_params__'):
                ret['source']['provider_params'][k.split('__')[-1]] = v
            elif k.startswith('source__'):
                ret['source'][k.split('__')[-1]] = v
            else:
                ret[k] = v

    return ret

def cli_inside_build(args):
    build_inside(input_method=args.input, input_args=args.input_arg,
                 substitutions=args.substitute)

class CLI(object):
    def __init__(self, formatter_class=argparse.HelpFormatter, prog=PROG):
        self.parser = argparse.ArgumentParser(
            prog=prog,
            description=DESCRIPTION,
            formatter_class=formatter_class,
        )
        self.build_parser = None
        self.bi_parser = None
        self.ib_parser = None

        locale.setlocale(locale.LC_ALL, '')

    def set_arguments(self):
        try:
            version = pkg_resources.get_distribution("atomic_reactor").version
        except pkg_resources.DistributionNotFound:
            version = "GIT"

        exclusive_group = self.parser.add_mutually_exclusive_group()
        exclusive_group.add_argument("-q", "--quiet", action="store_true")
        exclusive_group.add_argument("-v", "--verbose", action="store_true")
        exclusive_group.add_argument("-V", "--version", action="version", version=version)

        subparsers = self.parser.add_subparsers(help='commands')

        # BUILDING IMAGES

        self.build_parser = subparsers.add_parser(
            'build',
            usage="%s [OPTIONS] build" % PROG,
            description='This command enables you to build images. '
                        'Currently, you can build images from git repo, local path or '
                        'a json file with full build specification. '
                        'There are several methods for performing the build: '
                        'inside a build container using docker from host, '
                        'inside a build container using new instance of docker, '
                        'or within current environment.'
        )

        def cli_build_with_source(args):
            if args.source__provider is None:
                self.build_parser.print_help()
            else:
                cli_build_image(args)

        self.build_parser.set_defaults(func=cli_build_with_source)
        self.generate_source_types_subparsers()

        # CREATE BUILD IMAGE

        self.bi_parser = subparsers.add_parser('create-build-image',
                                               usage="%s [OPTIONS] create-build-image" % PROG,
                                               description='Create build image; Atomic Reactor '
                                               'installs itself inside and is capable of '
                                               'building images within this image.')
        self.bi_parser.set_defaults(func=cli_create_build_image)
        reactor_source = self.bi_parser.add_mutually_exclusive_group()
        reactor_source.add_argument("--reactor-latest", action='store_true',
                                    help="put latest Atomic Reactor inside (from public git)")
        reactor_source.add_argument("--reactor-remote-git", action='store',
                                    help="URL to git repo with Atomic Reactor (has to contain "
                                    "setup.py)")
        reactor_source.add_argument("--reactor-local-path", action='store',
                                    help="path to directory with Atomic Reactor (has to contain "
                                    "setup.py)")
        reactor_source.add_argument("--reactor-tarball-path", action='store',
                                    help="path to distribution tarball with Atomic Reactor")
        self.bi_parser.add_argument("dockerfile_dir_path", action="store",
                                    metavar="DOCKERFILE_DIR_PATH",
                                    help="path to directory with Dockerfile")
        self.bi_parser.add_argument("image", action='store', metavar="IMAGE",
                                    help="name under the image will be accessible")
        self.bi_parser.add_argument("--use-cache", action='store_true', default=False,
                                    help="use cache to build image (may be faster, but not up "
                                    "to date)")

        # inside build
        self.ib_parser = subparsers.add_parser(
            'inside-build',
            usage="%s [OPTIONS] inside-build" % PROG,
            description="build inside a container, taking build JSON input "
                        "from the source specified by the --input option.")
        self.ib_parser.add_argument("--input", action='store', default="auto",
                                    help="input plugin name (determined automatically unless "
                                    "given)")
        self.ib_parser.add_argument("--input-arg", action='append',
                                    help="argument for input plugin (in form of 'key=value'), "
                                    "see input plugins to know what arguments they accept (can "
                                    "be specified multiple times)")
        self.ib_parser.add_argument("--dont-pull-base-image", action='store_true',
                                    help="don't pull or update base image specified in "
                                    "dockerfile")
        self.ib_parser.add_argument("--substitute", action='append',
                                    help="substitute values in build json (key=value, or "
                                         "plugin_type.plugin_name.key=value)")
        self.ib_parser.set_defaults(func=cli_inside_build)

    def generate_source_types_subparsers(self):
        build_subparsers = self.build_parser.add_subparsers(help='select source provider to use',
                                                            dest='source__provider')
        self.source_types_parsers = {
            'git': build_subparsers.add_parser(
                'git',
                help='build from a git repo',
                usage="%s [OPTIONS] build git" % PROG),
            'path': build_subparsers.add_parser(
                'path',
                help='build from a local path',
                usage="%s [OPTIONS] build path" % PROG),
        }
        source_types_uri_help = {
            'git': 'URI of the git repo',
            'path': 'URI of the directory with source (e.g. "file:///foo/bar") or\
                    path (e.g. "/foo/bar"); path can be relative',
        }
        source_types_df_path_help = {
            'git': 'path to Dockerfile within git repo (default is ./)',
            'path': 'path to Dockerfile within given directory (default is ./)',
        }

        # add arguments that are common to non-json source types
        for st, stp in self.source_types_parsers.items():
            stp.add_argument(
                "--image", action='store', required=True,
                help="name under which the image will be accessible")
            stp.add_argument(
                "--uri", action='store', metavar="URI", required=True,
                help=source_types_uri_help[st], dest='source__uri')
            stp.add_argument(
                "--df-path", action='store', help=source_types_df_path_help[st],
                dest='source__dockerfile_path')
            stp.add_argument(
                "--source-registry", action='store', metavar="REGISTRY",
                help="registry to pull base image from")
            stp.add_argument(
                "--source-registry-insecure", action='store_true',
                help="allow connecting to source registry over plain http")
            stp.add_argument(
                "--target-registries", action='store', nargs="*", metavar="REGISTRY",
                help="list of registries to push image to")
            stp.add_argument(
                "--target-registries-insecure", action='store_true',
                help="allow connecting to target registries over plain http")
            stp.add_argument(
                "--dont-pull-base-image", action='store_true',
                help="don't pull or update base image specified in dockerfile")

        # add the "json" subparser that allows to build from a json build file
        self.source_types_parsers['json'] = build_subparsers.add_parser(
            'json',
            help='load build configuration from json file',
            usage="%s [OPTIONS] build json" % PROG)

        # add arguments common to both non-json and json source types
        for stp in self.source_types_parsers.values():
            stp.add_argument(
                "--build-image", action='store',
                help="name of build image to use (build image type has to match method)")
            stp.add_argument(
                "--load-plugin", action="store", nargs="*", metavar="PLUGIN_FILE",
                dest="plugin_files", help="list of files where plugins live")
            stp.add_argument(
                "--method", action='store', required=True,
                choices=["hostdocker", "privileged", "here"],
                help="choose method for building image: "
                     "'hostdocker' mounts socket inside privileged container to use docker from "
                     "host, 'privileged' runs separate docker instance inside privileged "
                     "container and 'here' executes build in current environment")

        # add sourcetype-specific arguments now
        self.source_types_parsers['git'].add_argument(
            '--commit', action='store',
            dest='source__provider_params__git_commit',
            help="checkout this commit (default is master)")
        self.source_types_parsers['json'].add_argument(
            'json_path', metavar='JSON_PATH',
            help='path to the build json')
        self.source_types_parsers['json'].add_argument(
            '--substitute', nargs='*', metavar='SUBSTITUTE',
            help='provide substitutions for json in form "foo.bar=spam"')

    def run(self):
        self.set_arguments()
        args = self.parser.parse_args()
        logging.captureWarnings(True)

        if args.verbose:
            set_logging(level=logging.DEBUG)
        elif args.quiet:
            set_logging(level=logging.WARNING)
        else:
            set_logging(level=logging.INFO)
        try:
            args.func(args)
        except AttributeError:
            if hasattr(args, 'func'):
                raise
            else:
                self.parser.print_help()
        except KeyboardInterrupt:
            pass
        except Exception as ex:
            if args.verbose:
                raise
            else:
                logger.error("exception caught: %r", ex)

def run():
    cli = CLI()
    cli.run()

if __name__ == '__main__':
    run()
<|endoftext|>


<commit_before>import os
from netCDF4 import Dataset as NCDataset
from dateutil import parser as dateparser

import logging
logger = logging.getLogger(__name__)

SPATIAL_VARIABLES =  [
    'longitude', 'lon',
    'latitude', 'lat',
    'altitude', 'alt', 'level', 'height',
    'rotated_pole',
    'rotated_latitude_longitude',
    'time']

class Dataset(object):
    def __init__(self, filepath):
        self.path = filepath
        self.name = os.path.basename(filepath)
        self.url = 'file://' + filepath
        self.content_type = 'application/netcdf'
        self.resourcename = filepath
        self.attributes = {}
        self._parse(filepath)

    def __str__(self):
        return "attributes={0}".format(self.attributes)

    @property
    def variable(self):
        return self.attributes.get('variable')

    @property
    def variable_long_name(self):
        return self.attributes.get('variable_long_name')

    @property
    def cf_standard_name(self):
        return self.attributes.get('cf_standard_name')

    @property
    def units(self):
        return self.attibutes.get('units')

    @property
    def comments(self):
        return self.attributes.get('comments')

    @property
    def institute(self):
        return self.attributes.get('institute_id')

    @property
    def experiment(self):
        return self.attributes.get('experiment_id')

    @property
    def project(self):
        return self.attibutes.get('project_id')

    @property
    def model(self):
        return self.attributes.get('model_id')

    @property
    def frequency(self):
        return self.attributes.get('frequency')

    @property
    def creation_date(self):
        if 'creation_date' in self.attributes:
            return self.attributes['creation_date'][0]
        else:
            return None

    @property
    def size(self):
        if 'size' in self.attributes:
            return self.attributes['size'][0]
        else:
            return ''

    @property
    def last_modified(self):
        return self.creation_date

    def _add_attribute(self, key, value):
        if not key in self.attributes:
            self.attributes[key] = []
        self.attributes[key].append(value)

    def _parse(self, filepath):
        filepath = os.path.abspath(filepath)
        logger.debug("parse %s", filepath)

        try:
            ds = NCDataset(filepath, 'r')

            # loop over global attributes
            for attname in ds.ncattrs():
                attvalue = getattr(ds, attname)
                if 'date' in attname.lower():
                    # must format dates in Solr format, if possible
                    try:
                        solr_dt = dateparser.parse(attvalue)
                        self._add_attribute(attname, solr_dt.strftime('%Y-%m-%dT%H:%M:%SZ') )
                    except:
                        pass # disregard this attribute
                else:
                    self._add_attribute(attname, attvalue)

            # loop over dimensions
            for key, dim in ds.dimensions.items():
                self._add_attribute('dimension', "%s:%s" % (key, len(dim)) )

            # loop over variable attributes
            for key, variable in ds.variables.items():
                if key.lower() in ds.dimensions:
                    # skip dimension variables
                    continue
                if '_bnds' in key.lower():
                    continue
                if key.lower() in SPATIAL_VARIABLES:
                    continue
                self._add_attribute('variable', key)
                self._add_attribute('variable_long_name', getattr(variable, 'long_name', None) )
                cf_standard_name = getattr(variable, 'standard_name', None)
                if cf_standard_name is not None:
                    self._add_attribute('cf_standard_name', getattr(variable, 'standard_name', None) )
                self._add_attribute('units', getattr(variable, 'units', None) )

        except Exception as e:
            logging.error(e)
        finally:
            try:
                ds.close()
            except:
                pass

def crawl(start_dir):
    if not os.path.isdir(start_dir):
        raise Exception("Invalid start directory: %s", start_dir)

    logger.info('start directory = %s', start_dir)

    for directory, subdirs, files in os.walk(start_dir):
        # loop over files in this directory
        for filename in files:
            # ignore hidden files and thumbnails
            if not filename[0] == '.' and not 'thumbnail' in filename and not filename.endswith('.xml'):
                filepath = os.path.join(directory, filename)
                yield Dataset(filepath)

<commit_msg>only crawl nc files<commit_after>import os
from netCDF4 import Dataset as NCDataset
from dateutil import parser as dateparser

import logging
logger = logging.getLogger(__name__)

SPATIAL_VARIABLES =  [
    'longitude', 'lon',
    'latitude', 'lat',
    'altitude', 'alt', 'level', 'height',
    'rotated_pole',
    'rotated_latitude_longitude',
    'time']

class Dataset(object):
    def __init__(self, filepath):
        self.path = filepath
        self.name = os.path.basename(filepath)
        self.url = 'file://' + filepath
        self.content_type = 'application/netcdf'
        self.resourcename = filepath
        self.attributes = {}
        self._parse(filepath)

    def __str__(self):
        return "attributes={0}".format(self.attributes)

    @property
    def variable(self):
        return self.attributes.get('variable')

    @property
    def variable_long_name(self):
        return self.attributes.get('variable_long_name')

    @property
    def cf_standard_name(self):
        return self.attributes.get('cf_standard_name')

    @property
    def units(self):
        return self.attibutes.get('units')

    @property
    def comments(self):
        return self.attributes.get('comments')

    @property
    def institute(self):
        return self.attributes.get('institute_id')

    @property
    def experiment(self):
        return self.attributes.get('experiment_id')

    @property
    def project(self):
        return self.attibutes.get('project_id')

    @property
    def model(self):
        return self.attributes.get('model_id')

    @property
    def frequency(self):
        return self.attributes.get('frequency')

    @property
    def creation_date(self):
        if 'creation_date' in self.attributes:
            return self.attributes['creation_date'][0]
        else:
            return None

    @property
    def size(self):
        if 'size' in self.attributes:
            return self.attributes['size'][0]
        else:
            return ''

    @property
    def last_modified(self):
        return self.creation_date

    def _add_attribute(self, key, value):
        if not key in self.attributes:
            self.attributes[key] = []
        self.attributes[key].append(value)

    def _parse(self, filepath):
        filepath = os.path.abspath(filepath)
        logger.debug("parse %s", filepath)

        try:
            ds = NCDataset(filepath, 'r')

            # loop over global attributes
            for attname in ds.ncattrs():
                attvalue = getattr(ds, attname)
                if 'date' in attname.lower():
                    # must format dates in Solr format, if possible
                    try:
                        solr_dt = dateparser.parse(attvalue)
                        self._add_attribute(attname, solr_dt.strftime('%Y-%m-%dT%H:%M:%SZ') )
                    except:
                        pass # disregard this attribute
                else:
                    self._add_attribute(attname, attvalue)

            # loop over dimensions
            for key, dim in ds.dimensions.items():
                self._add_attribute('dimension', "%s:%s" % (key, len(dim)) )

            # loop over variable attributes
            for key, variable in ds.variables.items():
                if key.lower() in ds.dimensions:
                    # skip dimension variables
                    continue
                if '_bnds' in key.lower():
                    continue
                if key.lower() in SPATIAL_VARIABLES:
                    continue
                self._add_attribute('variable', key)
                self._add_attribute('variable_long_name', getattr(variable, 'long_name', None) )
                cf_standard_name = getattr(variable, 'standard_name', None)
                if cf_standard_name is not None:
                    self._add_attribute('cf_standard_name', getattr(variable, 'standard_name', None) )
                self._add_attribute('units', getattr(variable, 'units', None) )

        except Exception as e:
            logging.error(e)
        finally:
            try:
                ds.close()
            except:
                pass

def crawl(start_dir):
    if not os.path.isdir(start_dir):
        raise Exception("Invalid start directory: %s", start_dir)

    logger.info('start directory = %s', start_dir)

    for directory, subdirs, files in os.walk(start_dir):
        # loop over files in this directory
        for filename in files:
            # only parse .nc files
            if filename.endswith('.nc'):
                filepath = os.path.join(directory, filename)
                yield Dataset(filepath)

<|endoftext|>


<commit_before>import audioBasicIO, sys, csv
import scipy.io.wavfile as wavfile

def annotation2files(wavFile, csvFile):
    '''
        Break an audio stream to segments of interest,
        defined by a csv file

        - wavFile:    path to input wavfile
        - csvFile:    path to csvFile of segment limits

        Input CSV file must be of the format <T1>\t<T2>\t<Label>
    '''

    [Fs, x] = audioBasicIO.readAudioFile(wavFile)
    with open(csvFile, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t', quotechar='|')
        for j, row in enumerate(reader):
            T1 = float(row[0].replace(",","."))
            T2 = float(row[1].replace(",","."))
            label = "%s_%s_%.2f_%.2f.wav" % (wavFile, row[2], T1, T2)
            label = label.replace(" ", "_")
            xtemp = x[round(T1*Fs):round(T2*Fs)]
            print T1, T2, label, xtemp.shape
            wavfile.write(label, Fs, xtemp)

def main(argv):
    wavFile = argv[1]
    annotationFile = argv[2]
    annotation2files(wavFile, annotationFile)

if __name__ == '__main__':
    # This file can be used to extract a series of annotated WAV files based on (a) an audio file (mp3 or wav) and
    # (b) a segment annotation file e.g. a "label" file generated in audacity
    # usage:
    # python audacityAnnotation2WAVs.py <audiofilepath> <annotationfilepath>
    #
    # The <annotationfilepath> is actually a tab-seperated file where each line has the format <startTime>\t<entTime>\t<classLabel>
    # The result of this process is a  series of WAV files with a file name <audiofilepath>_<startTime>_<endTime>_<classLabel>
    main(sys.argv)
    <commit_msg>audacityAnnotation2WAVs.py extract series of annotated wav files based on audacity annotations and a respective mp3 or wav recording<commit_after>import glob, os
import audioBasicIO, sys, csv
import scipy.io.wavfile as wavfile

def annotation2files(wavFile, csvFile):
    '''
        Break an audio stream to segments of interest,
        defined by a csv file

        - wavFile:    path to input wavfile
        - csvFile:    path to csvFile of segment limits

        Input CSV file must be of the format <T1>\t<T2>\t<Label>
    '''

    [Fs, x] = audioBasicIO.readAudioFile(wavFile)
    with open(csvFile, 'rb') as csvfile:
        reader = csv.reader(csvfile, delimiter='\t', quotechar='|')
        for j, row in enumerate(reader):
            T1 = float(row[0].replace(",","."))
            T2 = float(row[1].replace(",","."))
            label = "%s_%s_%.2f_%.2f.wav" % (wavFile, row[2], T1, T2)
            label = label.replace(" ", "_")
            xtemp = x[round(T1*Fs):round(T2*Fs)]
            print T1, T2, label, xtemp.shape
            wavfile.write(label, Fs, xtemp)

def main(argv):
    if argv[1] == "-f":
        wavFile = argv[2]
        annotationFile = argv[3]
        annotation2files(wavFile, annotationFile)
    elif argv[1] == "-d":
        inputFolder = argv[2]
        types = ('*.txt', '*.csv')
        annotationFilesList = []
        for files in types:
            annotationFilesList.extend(glob.glob(os.path.join(inputFolder, files)))
        for anFile in annotationFilesList:
            wavFile = os.path.splitext(anFile)[0] + ".wav"
            if not os.path.isfile(wavFile):
                wavFile = os.path.splitext(anFile)[0] + ".mp3"
                if not os.path.isfile(wavFile):
                    print "Audio file not found!"
                    return
            annotation2files(wavFile, anFile)

if __name__ == '__main__':
    # Used to extract a series of annotated WAV files based on (a) an audio file (mp3 or wav) and
    # (b) a segment annotation file e.g. a "label" file generated in audacity
    #
    # usage 1:
    # python audacityAnnotation2WAVs.py -f <audiofilepath> <annotationfilepath>
    # The <annotationfilepath> is actually a tab-seperated file where each line has the format <startTime>\t<entTime>\t<classLabel>
    # The result of this process is a  series of WAV files with a file name <audiofilepath>_<startTime>_<endTime>_<classLabel>
    #
    # usage 2:
    # python audacityAnnotation2WAVs.py -d <annotationfolderpath>
    # Same but searches all .txt and .csv annotation files. Audio files are supposed to be in the same path / filename with a WAV extension

    main(sys.argv)
    <|endoftext|>


<commit_before><commit_msg>Defined conditional serialization on results of `action`s.<commit_after><|endoftext|>