<commit_before># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2017  Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.

import asyncio
import errno
import functools
import io
import os
import shutil
import socket
import struct
import traceback

import qubes.exc

class ProtocolError(AssertionError):
    '''Raised when something is wrong with data received'''
    pass

class PermissionDenied(Exception):
    '''Raised deliberately by handlers when we decide not to cooperate'''
    pass

def method(name, *, no_payload=False, endpoints=None, **classifiers):
    '''Decorator factory for methods intended to appear in API.

    The decorated method can be called from public API using a child of
    :py:class:`AbstractQubesMgmt` class. The method becomes "public", and can be
    called using remote management interface.

    :param str name: qrexec rpc method name
    :param bool no_payload: if :py:obj:`True`, will barf on non-empty payload; \
        also will not pass payload at all to the method
    :param iterable endpoints: if specified, method serve multiple API calls
        generated by replacing `{endpoint}` with each value in this iterable

    The expected function method should have one argument (other than usual
    *self*), ``untrusted_payload``, which will contain the payload.

    .. warning::
        This argument has to be named such, to remind the programmer that the
        content of this variable is indeed untrusted.

    If *no_payload* is true, then the method is called with no arguments.
    '''

    def decorator(func):
        if no_payload:
            # the following assignment is needed for how closures work in Python
            _func = func
            @functools.wraps(_func)
            def wrapper(self, untrusted_payload, **kwargs):
                if untrusted_payload != b'':
                    raise ProtocolError('unexpected payload')
                return _func(self, **kwargs)
            func = wrapper

        # pylint: disable=protected-access
        if endpoints is None:
            func.rpcnames = ((name, None),)
        else:
            func.rpcnames = tuple(
                (name.format(endpoint=endpoint), endpoint)
                for endpoint in endpoints)

        func.classifiers = classifiers

        return func

    return decorator

def apply_filters(iterable, filters):
    '''Apply filters returned by admin-permission:... event'''
    for selector in filters:
        iterable = filter(selector, iterable)
    return iterable

class AbstractQubesAPI(object):
    '''Common code for Qubes Management Protocol handling

    Different interfaces can expose different API call sets, however they share
    common protocol and common implementation framework. This class is the
    latter.

    To implement a new interface, inherit from this class and write at least one
    method and decorate it with :py:func:`api` decorator. It will have access to
    pre-defined attributes: :py:attr:`app`, :py:attr:`src`, :py:attr:`dest`,
    :py:attr:`arg` and :py:attr:`method`.

    There are also two helper functions for firing events associated with API
    calls.
    '''

    #: the preferred socket location (to be overridden in child's class)
    SOCKNAME = None

    def __init__(self, app, src, method_name, dest, arg, send_event=None):
        #: :py:class:`qubes.Qubes` object
        self.app = app

        #: source qube
        self.src = self.app.domains[src.decode('ascii')]

        #: destination qube
        self.dest = self.app.domains[dest.decode('ascii')]

        #: argument
        self.arg = arg.decode('ascii')

        #: name of the method
        self.method = method_name.decode('ascii')

        #: callback for sending events if applicable
        self.send_event = send_event

        #: is this operation cancellable?
        self.cancellable = False

        candidates = list(self.list_methods(self.method))

        if not candidates:
            raise ProtocolError('no such method: {!r}'.format(self.method))

        assert len(candidates) == 1, \
            'multiple candidates for method {!r}'.format(self.method)

        #: the method to execute
        self._handler = candidates[0]
        self._running_handler = None

    @classmethod
    def list_methods(cls, select_method=None):
        for attr in dir(cls):
            func = getattr(cls, attr)
            if not callable(func):
                continue

            try:
                # pylint: disable=protected-access
                rpcnames = func.rpcnames
            except AttributeError:
                continue

            for mname, endpoint in rpcnames:
                if select_method is None or mname == select_method:
                    yield (func, mname, endpoint)

    def execute(self, *, untrusted_payload):
        '''Execute management operation.

        This method is a coroutine.
        '''
        handler, _, endpoint = self._handler
        kwargs = {}
        if endpoint is not None:
            kwargs['endpoint'] = endpoint
        self._running_handler = asyncio.ensure_future(handler(self,
            untrusted_payload=untrusted_payload, **kwargs))
        return self._running_handler

    def cancel(self):
        '''If operation is cancellable, interrupt it'''
        if self.cancellable and self._running_handler is not None:
            self._running_handler.cancel()

    def fire_event_for_permission(self, **kwargs):
        '''Fire an event on the source qube to check for permission'''
        return self.src.fire_event('admin-permission:' + self.method,
            pre_event=True, dest=self.dest, arg=self.arg, **kwargs)

    def fire_event_for_filter(self, iterable, **kwargs):
        '''Fire an event on the source qube to filter for permission'''
        return apply_filters(iterable,
            self.fire_event_for_permission(**kwargs))

    def enforce(self, predicate):
        '''An assert replacement, but works even with optimisations.'''
        if not predicate:
            raise PermissionDenied()

class QubesDaemonProtocol(asyncio.Protocol):
    buffer_size = 65536
    header = struct.Struct('Bx')
    # keep track of connections, to gracefully close them at server exit
    # (including cleanup of integration test)
    connections = set()

    def __init__(self, handler, *args, app, debug=False, **kwargs):
        super().__init__(*args, **kwargs)
        self.handler = handler
        self.app = app
        self.untrusted_buffer = io.BytesIO()
        self.len_untrusted_buffer = 0
        self.transport = None
        self.debug = debug
        self.event_sent = False
        self.mgmt = None

    def connection_made(self, transport):
        self.transport = transport
        self.connections.add(self)

    def connection_lost(self, exc):
        self.untrusted_buffer.close()
        # for cancellable operation, interrupt it, otherwise it will do nothing
        if self.mgmt is not None:
            self.mgmt.cancel()
        self.transport = None
        self.connections.remove(self)

    def data_received(self, untrusted_data):  # pylint: disable=arguments-differ
        if self.len_untrusted_buffer + len(untrusted_data) > self.buffer_size:
            self.app.log.warning('request too long')
            self.transport.abort()
            self.untrusted_buffer.close()
            return

        self.len_untrusted_buffer += \
            self.untrusted_buffer.write(untrusted_data)

    def eof_received(self):
        try:
            src, meth, dest, arg, untrusted_payload = \
                self.untrusted_buffer.getvalue().split(b'\0', 4)
        except ValueError:
            self.app.log.warning('framing error')
            self.transport.abort()
            return None
        finally:
            self.untrusted_buffer.close()

        asyncio.ensure_future(self.respond(
            src, meth, dest, arg, untrusted_payload=untrusted_payload))

        return True

    @asyncio.coroutine
    def respond(self, src, meth, dest, arg, *, untrusted_payload):
        try:
            self.mgmt = self.handler(self.app, src, meth, dest, arg,
                self.send_event)
            response = yield from self.mgmt.execute(
                untrusted_payload=untrusted_payload)
            assert not (self.event_sent and response)
            if self.transport is None:
                return

        # except clauses will fall through to transport.abort() below

        except PermissionDenied:
            self.app.log.warning(
                'permission denied for call %s+%s (%s → %s) '
                'with payload of %d bytes',
                    meth, arg, src, dest, len(untrusted_payload))

        except ProtocolError:
            self.app.log.warning(
                'protocol error for call %s+%s (%s → %s) '
                'with payload of %d bytes',
                    meth, arg, src, dest, len(untrusted_payload))

        except qubes.exc.QubesException as err:
            msg = ('%r while calling '
                'src=%r meth=%r dest=%r arg=%r len(untrusted_payload)=%d')

            if self.debug:
                self.app.log.debug(msg,
                    err, src, meth, dest, arg, len(untrusted_payload),
                    exc_info=1)
            if self.transport is not None:
                self.send_exception(err)
                self.transport.write_eof()
                self.transport.close()
            return

        except Exception:  # pylint: disable=broad-except
            self.app.log.exception(
                'unhandled exception while calling '
                'src=%r meth=%r dest=%r arg=%r len(untrusted_payload)=%d',
                    src, meth, dest, arg, len(untrusted_payload))

        else:
            if not self.event_sent:
                self.send_response(response)
            try:
                self.transport.write_eof()
            except NotImplementedError:
                pass
            self.transport.close()
            return

        # this is reached if from except: blocks; do not put it in finally:,
        # because this will prevent the good case from sending the reply
        if self.transport:
            self.transport.abort()

    def send_header(self, *args):
        self.transport.write(self.header.pack(*args))

    def send_response(self, content):
        assert not self.event_sent
        self.send_header(0x30)
        if content is not None:
            self.transport.write(content.encode('utf-8'))

    def send_event(self, subject, event, **kwargs):
        if self.transport is None:
            return
        self.event_sent = True
        self.send_header(0x31)

        if subject is not self.app:
            self.transport.write(str(subject).encode('ascii'))
        self.transport.write(b'\0')

        self.transport.write(event.encode('ascii') + b'\0')

        for k, v in kwargs.items():
            self.transport.write('{}\0{}\0'.format(k, str(v)).encode('ascii'))
        self.transport.write(b'\0')

    def send_exception(self, exc):
        self.send_header(0x32)

        self.transport.write(type(exc).__name__.encode() + b'\0')

        if self.debug:
            self.transport.write(''.join(traceback.format_exception(
                type(exc), exc, exc.__traceback__)).encode('utf-8'))
        self.transport.write(b'\0')

        self.transport.write(str(exc).encode('utf-8') + b'\0')

def cleanup_socket(sockpath, force):
    '''Remove socket if stale, or force=True
    :param sockpath: path to a socket
    :param force: should remove even if still used
    '''
    if force:
        os.unlink(sockpath)
    else:
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        try:
            sock.connect(sockpath)
        except ConnectionRefusedError:
            # dead socket, remove it anyway
            os.unlink(sockpath)
        else:
            # woops, someone is listening
            sock.close()
            raise FileExistsError(errno.EEXIST,
                'socket already exists: {!r}'.format(sockpath))

@asyncio.coroutine
def create_servers(*args, force=False, loop=None, **kwargs):
    '''Create multiple Qubes API servers

    :param qubes.Qubes app: the app that is a backend of the servers
    :param bool force: if :py:obj:`True`, unconditionally remove existing \
        sockets; if :py:obj:`False`, raise an error if there is some process \
        listening to such socket
    :param asyncio.Loop loop: loop

    *args* are supposed to be classes inheriting from
    :py:class:`AbstractQubesAPI`

    *kwargs* (like *app* or *debug* for example) are passed to
    :py:class:`QubesDaemonProtocol` constructor
    '''
    loop = loop or asyncio.get_event_loop()

    servers = []
    old_umask = os.umask(0o007)
    try:
        # XXX this can be optimised with asyncio.wait() to start servers in
        # parallel, but I currently don't see the need
        for handler in args:
            sockpath = handler.SOCKNAME
            assert sockpath is not None, \
                'SOCKNAME needs to be overloaded in {}'.format(
                    type(handler).__name__)

            if os.path.exists(sockpath):
                cleanup_socket(sockpath, force)

            server = yield from loop.create_unix_server(
                functools.partial(QubesDaemonProtocol, handler, **kwargs),
                sockpath)

            for sock in server.sockets:
                shutil.chown(sock.getsockname(), group='qubes')

            servers.append(server)
    except:
        for server in servers:
            for sock in server.sockets:
                try:
                    os.unlink(sock.getsockname())
                except FileNotFoundError:
                    pass
            server.close()
        if servers:
            yield from asyncio.wait([
                server.wait_closed() for server in servers])
        raise
    finally:
        os.umask(old_umask)

    return servers
<commit_msg>api: make enforce() static<commit_after># -*- encoding: utf8 -*-
#
# The Qubes OS Project, http://www.qubes-os.org
#
# Copyright (C) 2017  Wojtek Porczyk <woju@invisiblethingslab.com>
# Copyright (C) 2017  Marek Marczykowski-Górecki
#                               <marmarek@invisiblethingslab.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, see <https://www.gnu.org/licenses/>.

import asyncio
import errno
import functools
import io
import os
import shutil
import socket
import struct
import traceback

import qubes.exc

class ProtocolError(AssertionError):
    '''Raised when something is wrong with data received'''
    pass

class PermissionDenied(Exception):
    '''Raised deliberately by handlers when we decide not to cooperate'''
    pass

def method(name, *, no_payload=False, endpoints=None, **classifiers):
    '''Decorator factory for methods intended to appear in API.

    The decorated method can be called from public API using a child of
    :py:class:`AbstractQubesMgmt` class. The method becomes "public", and can be
    called using remote management interface.

    :param str name: qrexec rpc method name
    :param bool no_payload: if :py:obj:`True`, will barf on non-empty payload; \
        also will not pass payload at all to the method
    :param iterable endpoints: if specified, method serve multiple API calls
        generated by replacing `{endpoint}` with each value in this iterable

    The expected function method should have one argument (other than usual
    *self*), ``untrusted_payload``, which will contain the payload.

    .. warning::
        This argument has to be named such, to remind the programmer that the
        content of this variable is indeed untrusted.

    If *no_payload* is true, then the method is called with no arguments.
    '''

    def decorator(func):
        if no_payload:
            # the following assignment is needed for how closures work in Python
            _func = func
            @functools.wraps(_func)
            def wrapper(self, untrusted_payload, **kwargs):
                if untrusted_payload != b'':
                    raise ProtocolError('unexpected payload')
                return _func(self, **kwargs)
            func = wrapper

        # pylint: disable=protected-access
        if endpoints is None:
            func.rpcnames = ((name, None),)
        else:
            func.rpcnames = tuple(
                (name.format(endpoint=endpoint), endpoint)
                for endpoint in endpoints)

        func.classifiers = classifiers

        return func

    return decorator

def apply_filters(iterable, filters):
    '''Apply filters returned by admin-permission:... event'''
    for selector in filters:
        iterable = filter(selector, iterable)
    return iterable

class AbstractQubesAPI(object):
    '''Common code for Qubes Management Protocol handling

    Different interfaces can expose different API call sets, however they share
    common protocol and common implementation framework. This class is the
    latter.

    To implement a new interface, inherit from this class and write at least one
    method and decorate it with :py:func:`api` decorator. It will have access to
    pre-defined attributes: :py:attr:`app`, :py:attr:`src`, :py:attr:`dest`,
    :py:attr:`arg` and :py:attr:`method`.

    There are also two helper functions for firing events associated with API
    calls.
    '''

    #: the preferred socket location (to be overridden in child's class)
    SOCKNAME = None

    def __init__(self, app, src, method_name, dest, arg, send_event=None):
        #: :py:class:`qubes.Qubes` object
        self.app = app

        #: source qube
        self.src = self.app.domains[src.decode('ascii')]

        #: destination qube
        self.dest = self.app.domains[dest.decode('ascii')]

        #: argument
        self.arg = arg.decode('ascii')

        #: name of the method
        self.method = method_name.decode('ascii')

        #: callback for sending events if applicable
        self.send_event = send_event

        #: is this operation cancellable?
        self.cancellable = False

        candidates = list(self.list_methods(self.method))

        if not candidates:
            raise ProtocolError('no such method: {!r}'.format(self.method))

        assert len(candidates) == 1, \
            'multiple candidates for method {!r}'.format(self.method)

        #: the method to execute
        self._handler = candidates[0]
        self._running_handler = None

    @classmethod
    def list_methods(cls, select_method=None):
        for attr in dir(cls):
            func = getattr(cls, attr)
            if not callable(func):
                continue

            try:
                # pylint: disable=protected-access
                rpcnames = func.rpcnames
            except AttributeError:
                continue

            for mname, endpoint in rpcnames:
                if select_method is None or mname == select_method:
                    yield (func, mname, endpoint)

    def execute(self, *, untrusted_payload):
        '''Execute management operation.

        This method is a coroutine.
        '''
        handler, _, endpoint = self._handler
        kwargs = {}
        if endpoint is not None:
            kwargs['endpoint'] = endpoint
        self._running_handler = asyncio.ensure_future(handler(self,
            untrusted_payload=untrusted_payload, **kwargs))
        return self._running_handler

    def cancel(self):
        '''If operation is cancellable, interrupt it'''
        if self.cancellable and self._running_handler is not None:
            self._running_handler.cancel()

    def fire_event_for_permission(self, **kwargs):
        '''Fire an event on the source qube to check for permission'''
        return self.src.fire_event('admin-permission:' + self.method,
            pre_event=True, dest=self.dest, arg=self.arg, **kwargs)

    def fire_event_for_filter(self, iterable, **kwargs):
        '''Fire an event on the source qube to filter for permission'''
        return apply_filters(iterable,
            self.fire_event_for_permission(**kwargs))

    @staticmethod
    def enforce(predicate):
        '''An assert replacement, but works even with optimisations.'''
        if not predicate:
            raise PermissionDenied()

class QubesDaemonProtocol(asyncio.Protocol):
    buffer_size = 65536
    header = struct.Struct('Bx')
    # keep track of connections, to gracefully close them at server exit
    # (including cleanup of integration test)
    connections = set()

    def __init__(self, handler, *args, app, debug=False, **kwargs):
        super().__init__(*args, **kwargs)
        self.handler = handler
        self.app = app
        self.untrusted_buffer = io.BytesIO()
        self.len_untrusted_buffer = 0
        self.transport = None
        self.debug = debug
        self.event_sent = False
        self.mgmt = None

    def connection_made(self, transport):
        self.transport = transport
        self.connections.add(self)

    def connection_lost(self, exc):
        self.untrusted_buffer.close()
        # for cancellable operation, interrupt it, otherwise it will do nothing
        if self.mgmt is not None:
            self.mgmt.cancel()
        self.transport = None
        self.connections.remove(self)

    def data_received(self, untrusted_data):  # pylint: disable=arguments-differ
        if self.len_untrusted_buffer + len(untrusted_data) > self.buffer_size:
            self.app.log.warning('request too long')
            self.transport.abort()
            self.untrusted_buffer.close()
            return

        self.len_untrusted_buffer += \
            self.untrusted_buffer.write(untrusted_data)

    def eof_received(self):
        try:
            src, meth, dest, arg, untrusted_payload = \
                self.untrusted_buffer.getvalue().split(b'\0', 4)
        except ValueError:
            self.app.log.warning('framing error')
            self.transport.abort()
            return None
        finally:
            self.untrusted_buffer.close()

        asyncio.ensure_future(self.respond(
            src, meth, dest, arg, untrusted_payload=untrusted_payload))

        return True

    @asyncio.coroutine
    def respond(self, src, meth, dest, arg, *, untrusted_payload):
        try:
            self.mgmt = self.handler(self.app, src, meth, dest, arg,
                self.send_event)
            response = yield from self.mgmt.execute(
                untrusted_payload=untrusted_payload)
            assert not (self.event_sent and response)
            if self.transport is None:
                return

        # except clauses will fall through to transport.abort() below

        except PermissionDenied:
            self.app.log.warning(
                'permission denied for call %s+%s (%s → %s) '
                'with payload of %d bytes',
                    meth, arg, src, dest, len(untrusted_payload))

        except ProtocolError:
            self.app.log.warning(
                'protocol error for call %s+%s (%s → %s) '
                'with payload of %d bytes',
                    meth, arg, src, dest, len(untrusted_payload))

        except qubes.exc.QubesException as err:
            msg = ('%r while calling '
                'src=%r meth=%r dest=%r arg=%r len(untrusted_payload)=%d')

            if self.debug:
                self.app.log.debug(msg,
                    err, src, meth, dest, arg, len(untrusted_payload),
                    exc_info=1)
            if self.transport is not None:
                self.send_exception(err)
                self.transport.write_eof()
                self.transport.close()
            return

        except Exception:  # pylint: disable=broad-except
            self.app.log.exception(
                'unhandled exception while calling '
                'src=%r meth=%r dest=%r arg=%r len(untrusted_payload)=%d',
                    src, meth, dest, arg, len(untrusted_payload))

        else:
            if not self.event_sent:
                self.send_response(response)
            try:
                self.transport.write_eof()
            except NotImplementedError:
                pass
            self.transport.close()
            return

        # this is reached if from except: blocks; do not put it in finally:,
        # because this will prevent the good case from sending the reply
        if self.transport:
            self.transport.abort()

    def send_header(self, *args):
        self.transport.write(self.header.pack(*args))

    def send_response(self, content):
        assert not self.event_sent
        self.send_header(0x30)
        if content is not None:
            self.transport.write(content.encode('utf-8'))

    def send_event(self, subject, event, **kwargs):
        if self.transport is None:
            return
        self.event_sent = True
        self.send_header(0x31)

        if subject is not self.app:
            self.transport.write(str(subject).encode('ascii'))
        self.transport.write(b'\0')

        self.transport.write(event.encode('ascii') + b'\0')

        for k, v in kwargs.items():
            self.transport.write('{}\0{}\0'.format(k, str(v)).encode('ascii'))
        self.transport.write(b'\0')

    def send_exception(self, exc):
        self.send_header(0x32)

        self.transport.write(type(exc).__name__.encode() + b'\0')

        if self.debug:
            self.transport.write(''.join(traceback.format_exception(
                type(exc), exc, exc.__traceback__)).encode('utf-8'))
        self.transport.write(b'\0')

        self.transport.write(str(exc).encode('utf-8') + b'\0')

def cleanup_socket(sockpath, force):
    '''Remove socket if stale, or force=True
    :param sockpath: path to a socket
    :param force: should remove even if still used
    '''
    if force:
        os.unlink(sockpath)
    else:
        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        try:
            sock.connect(sockpath)
        except ConnectionRefusedError:
            # dead socket, remove it anyway
            os.unlink(sockpath)
        else:
            # woops, someone is listening
            sock.close()
            raise FileExistsError(errno.EEXIST,
                'socket already exists: {!r}'.format(sockpath))

@asyncio.coroutine
def create_servers(*args, force=False, loop=None, **kwargs):
    '''Create multiple Qubes API servers

    :param qubes.Qubes app: the app that is a backend of the servers
    :param bool force: if :py:obj:`True`, unconditionally remove existing \
        sockets; if :py:obj:`False`, raise an error if there is some process \
        listening to such socket
    :param asyncio.Loop loop: loop

    *args* are supposed to be classes inheriting from
    :py:class:`AbstractQubesAPI`

    *kwargs* (like *app* or *debug* for example) are passed to
    :py:class:`QubesDaemonProtocol` constructor
    '''
    loop = loop or asyncio.get_event_loop()

    servers = []
    old_umask = os.umask(0o007)
    try:
        # XXX this can be optimised with asyncio.wait() to start servers in
        # parallel, but I currently don't see the need
        for handler in args:
            sockpath = handler.SOCKNAME
            assert sockpath is not None, \
                'SOCKNAME needs to be overloaded in {}'.format(
                    type(handler).__name__)

            if os.path.exists(sockpath):
                cleanup_socket(sockpath, force)

            server = yield from loop.create_unix_server(
                functools.partial(QubesDaemonProtocol, handler, **kwargs),
                sockpath)

            for sock in server.sockets:
                shutil.chown(sock.getsockname(), group='qubes')

            servers.append(server)
    except:
        for server in servers:
            for sock in server.sockets:
                try:
                    os.unlink(sock.getsockname())
                except FileNotFoundError:
                    pass
            server.close()
        if servers:
            yield from asyncio.wait([
                server.wait_closed() for server in servers])
        raise
    finally:
        os.umask(old_umask)

    return servers
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Created on Thu May 23 16:35:42 2013

@author: dcriger
"""

from operator import add

from numpy.core.multiarray import array
from numpy.core.shape_base import hstack
from numpy.matrixlib.defmatrix import matrix
from numpy import sqrt
from scipy.linalg import eig

from qutip.superoperator import vec2mat
from qutip.qobj import Qobj

def _dep_super(pe):
    """
    Returns the superoperator corresponding to qubit depolarization for a
    given parameter pe.

    # TODO if this is going into production (hopefully it isn't) then check
    CPTP, expand to arbitrary dimensional systems, etc.
    """
    return Qobj(dims=[[2, 2], [2, 2]],
                inpt=array([[1. - pe / 2., 0., 0., pe / 2.],
                            [0., 1. - pe, 0., 0.],
                            [0., 0., 1. - pe, 0.],
                            [pe / 2., 0., 0., 1. - pe / 2.]]))

def _dep_choi(pe):
    """
    Returns the choi matrix corresponding to qubit depolarization for a
    given parameter pe.

    # TODO if this is going into production (hopefully it isn't) then check
    CPTP, expand to arbitrary dimensional systems, etc.
    """
    return Qobj(dims=[[2, 2], [2, 2]],
                inpt=array([[1. - pe / 2., 0., 0., 1. - pe],
                            [0., pe / 2., 0., 0.],
                            [0., 0., pe / 2., 0.],
                            [1. - pe, 0., 0., 1. - pe / 2.]]))

#- PRIVATE CONVERSION FUNCTIONS -----------------------------------------------
# These functions handle the main work of converting between representations,
# and are exposed below by other functions that add postconditions about types.
#
# TODO: handle type='kraus' as a three-index Qobj, rather than as a list?

def _super_tofrom_choi(q_oper):
    """
    We exploit that the basis transformation between Choi and supermatrix
    representations squares to the identity, so that if we munge Qobj.type,
    we can use the same function.

    Since this function doesn't respect :attr:`Qobj.type`, we mark it as
    private; only those functions which wrap this in a way so as to preserve
    type should be called externally.
    """
    data = q_oper.data.toarray()
    sqrt_shape = sqrt(data.shape[0])
    return Qobj(dims=q_oper.dims,
                inpt=data.reshape([sqrt_shape] * 4).
                transpose(3, 1, 2, 0).reshape(q_oper.data.shape))

def super_to_choi(q_oper):
    # TODO: deprecate and make private in favor of to_choi,
    #       which looks at Qobj.type to determine the right conversion function.
    """
    Takes a superoperator to a Choi matrix
    # TODO Sanitize input, incorporate as method on Qobj if type=='super'
    """
    q_oper = _super_tofrom_choi(q_oper)
    q_oper.type = 'choi'
    return q_oper

def choi_to_super(q_oper):
    # TODO: deprecate and make private in favor of to_super,
    #       which looks at Qobj.type to determine the right conversion function.
    """
    Takes a Choi matrix to a superoperator
    # TODO Sanitize input, Abstract-ify application of channels to states
    """
    q_oper = super_to_choi(q_oper)
    q_oper.type = 'super'
    return q_oper

def choi_to_kraus(q_oper):
    """
    Takes a Choi matrix and returns a list of Kraus operators.
    # TODO Create a new class structure for quantum channels, perhaps as a
    strict sub-class of Qobj.
    """
    vals, vecs = eig(q_oper.data.todense())
    vecs = list(map(array, zip(*vecs)))
    return list(map(lambda x: Qobj(inpt=x),
                    [sqrt(vals[j]) * vec2mat(vecs[j])
                     for j in range(len(vals))]))

def kraus_to_choi(kraus_list):
    """
    Takes a list of Kraus operators and returns the Choi matrix for the channel
    represented by the Kraus operators in `kraus_list`
    """
    kraus_mat_list = list(map(lambda x: matrix(x.data.todense()), kraus_list))
    op_len = len(kraus_mat_list[0])
    op_rng = range(op_len)
    choi_blocks = array([[sum([op[:, c_ix] * array([op.H[r_ix, :]])
                               for op in kraus_mat_list])
                          for r_ix in op_rng]
                         for c_ix in op_rng])
    return Qobj(inpt=hstack(hstack(choi_blocks)),
                dims=[kraus_list[0].dims, kraus_list[0].dims])

def kraus_to_super(kraus_list):
    return choi_to_super(kraus_to_choi(kraus_list))

#- PUBLIC CONVERSION FUNCTIONS ------------------------------------------------
# These functions handle superoperator conversions in a way that preserves the
# correctness of Qobj.type, and in a way that automatically branches based on
# the input Qobj.type.

def to_choi(q_oper):
    # TODO: docstring
    if q_oper.type == 'choi':
        return q_oper
    elif q_oper.type == 'super':
        return choi_to_super(q_oper)
    elif q_oper.type = 'oper':
        return choi_to_super(spre(q_oper) * spost(q_oper.dag()))
    else:
        raise TypeError("Conversion of Qobj.type = {} to Choi not supported.".format(q_oper.type))

<commit_msg>Fixed silly mistakes in previous commit.<commit_after># -*- coding: utf-8 -*-
"""
Created on Thu May 23 16:35:42 2013

@author: dcriger
"""

from operator import add

from numpy.core.multiarray import array
from numpy.core.shape_base import hstack
from numpy.matrixlib.defmatrix import matrix
from numpy import sqrt
from scipy.linalg import eig

from qutip.superoperator import vec2mat, spre, spost
from qutip.qobj import Qobj

def _dep_super(pe):
    """
    Returns the superoperator corresponding to qubit depolarization for a
    given parameter pe.

    # TODO if this is going into production (hopefully it isn't) then check
    CPTP, expand to arbitrary dimensional systems, etc.
    """
    return Qobj(dims=[[2, 2], [2, 2]],
                inpt=array([[1. - pe / 2., 0., 0., pe / 2.],
                            [0., 1. - pe, 0., 0.],
                            [0., 0., 1. - pe, 0.],
                            [pe / 2., 0., 0., 1. - pe / 2.]]))

def _dep_choi(pe):
    """
    Returns the choi matrix corresponding to qubit depolarization for a
    given parameter pe.

    # TODO if this is going into production (hopefully it isn't) then check
    CPTP, expand to arbitrary dimensional systems, etc.
    """
    return Qobj(dims=[[2, 2], [2, 2]],
                inpt=array([[1. - pe / 2., 0., 0., 1. - pe],
                            [0., pe / 2., 0., 0.],
                            [0., 0., pe / 2., 0.],
                            [1. - pe, 0., 0., 1. - pe / 2.]]))

#- PRIVATE CONVERSION FUNCTIONS -----------------------------------------------
# These functions handle the main work of converting between representations,
# and are exposed below by other functions that add postconditions about types.
#
# TODO: handle type='kraus' as a three-index Qobj, rather than as a list?

def _super_tofrom_choi(q_oper):
    """
    We exploit that the basis transformation between Choi and supermatrix
    representations squares to the identity, so that if we munge Qobj.type,
    we can use the same function.

    Since this function doesn't respect :attr:`Qobj.type`, we mark it as
    private; only those functions which wrap this in a way so as to preserve
    type should be called externally.
    """
    data = q_oper.data.toarray()
    sqrt_shape = sqrt(data.shape[0])
    return Qobj(dims=q_oper.dims,
                inpt=data.reshape([sqrt_shape] * 4).
                transpose(3, 1, 2, 0).reshape(q_oper.data.shape))

def super_to_choi(q_oper):
    # TODO: deprecate and make private in favor of to_choi,
    #       which looks at Qobj.type to determine the right conversion function.
    """
    Takes a superoperator to a Choi matrix
    # TODO Sanitize input, incorporate as method on Qobj if type=='super'
    """
    q_oper = _super_tofrom_choi(q_oper)
    q_oper.type = 'choi'
    return q_oper

def choi_to_super(q_oper):
    # TODO: deprecate and make private in favor of to_super,
    #       which looks at Qobj.type to determine the right conversion function.
    """
    Takes a Choi matrix to a superoperator
    # TODO Sanitize input, Abstract-ify application of channels to states
    """
    q_oper = super_to_choi(q_oper)
    q_oper.type = 'super'
    return q_oper

def choi_to_kraus(q_oper):
    """
    Takes a Choi matrix and returns a list of Kraus operators.
    # TODO Create a new class structure for quantum channels, perhaps as a
    strict sub-class of Qobj.
    """
    vals, vecs = eig(q_oper.data.todense())
    vecs = list(map(array, zip(*vecs)))
    return list(map(lambda x: Qobj(inpt=x),
                    [sqrt(vals[j]) * vec2mat(vecs[j])
                     for j in range(len(vals))]))

def kraus_to_choi(kraus_list):
    """
    Takes a list of Kraus operators and returns the Choi matrix for the channel
    represented by the Kraus operators in `kraus_list`
    """
    kraus_mat_list = list(map(lambda x: matrix(x.data.todense()), kraus_list))
    op_len = len(kraus_mat_list[0])
    op_rng = range(op_len)
    choi_blocks = array([[sum([op[:, c_ix] * array([op.H[r_ix, :]])
                               for op in kraus_mat_list])
                          for r_ix in op_rng]
                         for c_ix in op_rng])
    return Qobj(inpt=hstack(hstack(choi_blocks)),
                dims=[kraus_list[0].dims, kraus_list[0].dims])

def kraus_to_super(kraus_list):
    return choi_to_super(kraus_to_choi(kraus_list))

#- PUBLIC CONVERSION FUNCTIONS ------------------------------------------------
# These functions handle superoperator conversions in a way that preserves the
# correctness of Qobj.type, and in a way that automatically branches based on
# the input Qobj.type.

def to_choi(q_oper):
    # TODO: docstring
    if q_oper.type == 'choi':
        return q_oper
    elif q_oper.type == 'super':
        return super_to_choi(q_oper)
    elif q_oper.type == 'oper':
        return super_to_choi(spre(q_oper) * spost(q_oper.dag()))
    else:
        raise TypeError("Conversion of Qobj.type = {} to Choi not supported.".format(q_oper.type))

<|endoftext|>


<commit_before>##
# import this unit to fixed multilanguage read /write problems,
# it's a rarely appear problem : even you set the correct locale,
# if there are a folder / file name is mix with different language copy
# from other windows PC, you could not read / write with the correct
# name !
#
# we have to do many jobs to fix it :
#
# 1. replace the stdout / stderr / stdin file object in sys with our fixed objects
# 2. fixed the sys.argv list with GetCommandLine() api in win32
# 3. fixed os.path series problems
# 4. may be there have more we have not found yet! ...
#
# @date 2011-1-25
# @author: HongShe Liang <starofrainnight@gmail.com>
#
import sys
import os
import io
import threading
import atexit
import win32console
import win32api
import win32file
import types # for all standard type values for buildin type()
import pickle
import rabird.windows_api
import time

# * replace the stdout / stderr / stdin file object in sys with our fixed objects

class stdout_thread_t(threading.Thread):
	def __init__(self, file_descriptor, std_handle_type, old_stdout):
		threading.Thread.__init__(self)
		# so that if the main thread exit, our thread will also exit
		self.setDaemon(True)
		self.file_descriptor = file_descriptor
		self.std_handle_type = std_handle_type
		self.old_stdout = old_stdout

		# use for internal purpose
		self.screen_buffer = win32console.GetStdHandle(self.std_handle_type)
		# do not open the file in text mode, otherwise it will try to decode
		# with the encoding, but the unicode sometimes included something
		# do not in the encoding scale, that will case a convertion error:
		# 'ascii' codec can't encode character u'\xbb' in position ...
		self.stdout = io.open(self.file_descriptor, mode='rb')

	def run(self):
		end_mark = ord(".")
		exit_mark = ord("@")

		while True:
			# it will be block here until any string coming ...
			# we shoudl be read three lines for a unit ( the pickle format )
			temp_line = ""
			a_line = ""

			while True:
				temp_line = self.stdout.readline()
				a_line += temp_line

				if len(temp_line) > 0:
					if ord(temp_line[0]) == end_mark:
						break

			# if outside need us to destroy our self, we exit ...
			if len(temp_line) >= 2:
				if ord(temp_line[1]) == exit_mark:
					break

			if not a_line :
				continue

			s = pickle.loads(a_line)
			if len(s) <= 0 :
				continue

			if(type(s) == types.UnicodeType) :
				if self.screen_buffer is not None:
					# While debuging in Aptana Studio or some other situation,
					# the screen buffer will contain an invalid handle lead to
					# win32console.error be throw. We will use the old stdout
					# to output our informations instead.
					try:
						self.screen_buffer.WriteConsole(s)
					except win32console.error:
						self.screen_buffer = None
				else:
					self.old_stdout.write(s)
			else :
				self.old_stdout.write(s)

class stdio_file_t(io.FileIO):
	def __init__(self, name, mode='r', closefd=True):
		io.FileIO.__init__(self, name, mode, closefd)

	def write(self, value):
		if(type(value) != types.UnicodeType) :
			value = str(value) # changed value to str

		# the line separator must be at the end of line !
		value = str(pickle.dumps(value)) + os.linesep
		io.FileIO.write(self, value)

def stop_stdout_thread( a_thread, a_stdout_file ):
	os.write( a_stdout_file.fileno(), ".@\n" ) # break the read line operation in thread
	a_thread.join()

stdout_pipe = os.pipe()
stderr_pipe = os.pipe()
stdin_pipe = os.pipe()

old_stdout = sys.stdout
old_stderr = sys.stderr
old_stdin = sys.stdin

stdout_thread = stdout_thread_t(stdout_pipe[0], win32api.STD_OUTPUT_HANDLE, old_stdout)
stderr_thread = stdout_thread_t(stderr_pipe[0], win32api.STD_ERROR_HANDLE, old_stderr)

stdout_thread.start()
stderr_thread.start()

# to test our stdout with new stdout
our_stdout = stdio_file_t(stdout_pipe[1], "wb")
our_stderr = stdio_file_t(stderr_pipe[1], "wb")

sys.stdout = our_stdout
# keep not to overwrite the sys.stderr for debug purpose
sys.stderr = our_stderr

# * fixed the sys.argv list with GetCommandLine() api in win32
sys.argv = rabird.windows_api.CommandLineToArgv(rabird.windows_api.GetCommandLine())

# * finalize windows's unicode fix
def __on_exit_rabird_module():
	# wait for console finished their output
	stop_stdout_thread( stdout_thread, our_stdout )
	stop_stdout_thread( stderr_thread, our_stderr )

	# restore original standard input/output
	sys.stderr = old_stderr
	sys.stdout = old_stdout
	sys.stdin = old_stdin

	# close all pipes
	for i in stdin_pipe:
		os.close(i)

# we must restore original standard input/output file
atexit.register(__on_exit_rabird_module)

<commit_msg>Fixed unicode string will not output after met an error throw from screen buffer object.<commit_after>##
# import this unit to fixed multilanguage read /write problems,
# it's a rarely appear problem : even you set the correct locale,
# if there are a folder / file name is mix with different language copy
# from other windows PC, you could not read / write with the correct
# name !
#
# we have to do many jobs to fix it :
#
# 1. replace the stdout / stderr / stdin file object in sys with our fixed objects
# 2. fixed the sys.argv list with GetCommandLine() api in win32
# 3. fixed os.path series problems
# 4. may be there have more we have not found yet! ...
#
# @date 2011-1-25
# @author: HongShe Liang <starofrainnight@gmail.com>
#
import sys
import os
import io
import threading
import atexit
import win32console
import win32api
import win32file
import types # for all standard type values for buildin type()
import pickle
import rabird.windows_api
import time

# * replace the stdout / stderr / stdin file object in sys with our fixed objects

class stdout_thread_t(threading.Thread):
	def __init__(self, file_descriptor, std_handle_type, old_stdout):
		threading.Thread.__init__(self)
		# so that if the main thread exit, our thread will also exit
		self.setDaemon(True)
		self.file_descriptor = file_descriptor
		self.std_handle_type = std_handle_type
		self.old_stdout = old_stdout

		# use for internal purpose
		self.screen_buffer = win32console.GetStdHandle(self.std_handle_type)
		# do not open the file in text mode, otherwise it will try to decode
		# with the encoding, but the unicode sometimes included something
		# do not in the encoding scale, that will case a convertion error:
		# 'ascii' codec can't encode character u'\xbb' in position ...
		self.stdout = io.open(self.file_descriptor, mode='rb')

	def run(self):
		end_mark = ord(".")
		exit_mark = ord("@")

		while True:
			# it will be block here until any string coming ...
			# we shoudl be read three lines for a unit ( the pickle format )
			temp_line = ""
			a_line = ""

			while True:
				temp_line = self.stdout.readline()
				a_line += temp_line

				if len(temp_line) > 0:
					if ord(temp_line[0]) == end_mark:
						break

			# if outside need us to destroy our self, we exit ...
			if len(temp_line) >= 2:
				if ord(temp_line[1]) == exit_mark:
					break

			if not a_line :
				continue

			s = pickle.loads(a_line)
			if len(s) <= 0 :
				continue

			if(type(s) == types.UnicodeType) :
				if self.screen_buffer is not None:
					# While debuging in Aptana Studio or some other situation,
					# the screen buffer will contain an invalid handle lead to
					# win32console.error be throw. We will use the old stdout
					# to output our informations instead.
					try:
						self.screen_buffer.WriteConsole(s)
					except win32console.error:
						self.screen_buffer = None

						# If we could not output through screen buffer, we
						# convert the unicode string to python native string,
						# and write directly to old stndard output.
						self.old_stdout.write(str(s))
				else:
					self.old_stdout.write(str(s))
			else :
				self.old_stdout.write(s)

class stdio_file_t(io.FileIO):
	def __init__(self, name, mode='r', closefd=True):
		io.FileIO.__init__(self, name, mode, closefd)

	def write(self, value):
		if(type(value) != types.UnicodeType) :
			value = str(value) # changed value to str

		# the line separator must be at the end of line !
		value = str(pickle.dumps(value)) + os.linesep
		io.FileIO.write(self, value)

def stop_stdout_thread( a_thread, a_stdout_file ):
	os.write( a_stdout_file.fileno(), ".@\n" ) # break the read line operation in thread
	a_thread.join()

stdout_pipe = os.pipe()
stderr_pipe = os.pipe()
stdin_pipe = os.pipe()

old_stdout = sys.stdout
old_stderr = sys.stderr
old_stdin = sys.stdin

stdout_thread = stdout_thread_t(stdout_pipe[0], win32api.STD_OUTPUT_HANDLE, old_stdout)
stderr_thread = stdout_thread_t(stderr_pipe[0], win32api.STD_ERROR_HANDLE, old_stderr)

stdout_thread.start()
stderr_thread.start()

# to test our stdout with new stdout
our_stdout = stdio_file_t(stdout_pipe[1], "wb")
our_stderr = stdio_file_t(stderr_pipe[1], "wb")

sys.stdout = our_stdout
# keep not to overwrite the sys.stderr for debug purpose
sys.stderr = our_stderr

# * fixed the sys.argv list with GetCommandLine() api in win32
sys.argv = rabird.windows_api.CommandLineToArgv(rabird.windows_api.GetCommandLine())

# * finalize windows's unicode fix
def __on_exit_rabird_module():
	# wait for console finished their output
	stop_stdout_thread( stdout_thread, our_stdout )
	stop_stdout_thread( stderr_thread, our_stderr )

	# restore original standard input/output
	sys.stderr = old_stderr
	sys.stdout = old_stdout
	sys.stdin = old_stdin

	# close all pipes
	for i in stdin_pipe:
		os.close(i)

# we must restore original standard input/output file
atexit.register(__on_exit_rabird_module)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

# __version__ = '0.5.0'
__version__ = '0.4.1'

FRAMEWORK_NAME = 'RAPyDo'

# PROJECT_YAML_SPECSDIR = 'specs'
COMPOSE_ENVIRONMENT_FILE = '.env'
SUBMODULES_DIR = 'submodules'
PLACEHOLDER = '#@$%-REPLACE-#@%$-ME-#@$%'
<commit_msg>Bump to release rapydo_controller-0.4.2.tar.gz<commit_after># -*- coding: utf-8 -*-

# __version__ = '0.5.0'
__version__ = '0.4.2'

FRAMEWORK_NAME = 'RAPyDo'

# PROJECT_YAML_SPECSDIR = 'specs'
COMPOSE_ENVIRONMENT_FILE = '.env'
SUBMODULES_DIR = 'submodules'
PLACEHOLDER = '#@$%-REPLACE-#@%$-ME-#@$%'
<|endoftext|>


<commit_before>#  File:       etl.py
#  Purpose:    To do the `Transform` step of an Extract-Transform-Load.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Thursday 22 September 2016, 03:40 PM

def transform(num):
    old = {
           1: ["A", "E", "I", "O", "U", "L", "N", "R", "S", "T"],
           2: ["D", "G"],
           3: ["B", "C", "M", "P"],
           4: ["F", "H", "V", "W", "Y"],
           5: ["K"],
           8: ["J", "X"],
           10: ["Q", "Z"]
    }
    return old[num]

print(transform(1))
<commit_msg>Partial output<commit_after>#  File:       etl.py
#  Purpose:    To do the `Transform` step of an Extract-Transform-Load.
#  Programmer: Amal Shehu
#  Course:     Exercism
#  Date:       Thursday 22 September 2016, 03:40 PM

def transform(words):
    letters = dict(words)
    for letter in letters.iteritem():
        return letter.lower()
<|endoftext|>


<commit_before>import csv
import tensorflow as tf
import numpy as np
from src.autoencoder import autoencoder

# Read in CSV file
brca = []
count = 0

# brca is 192 long
with open("brca_toronto_collab_mutect_123_030617.csv") as csvFile:
    reader = csv.reader(csvFile)
    for row in reader:
        row.pop()
        if count != 0:
            floatRow = [float(i) for i in row]
            brca.append(floatRow)
        count += 1
csvFile.close()

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer())
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer())
y11 = tf.nn.relu(tf.matmul(x11, W11) + b11)
y12 = tf.nn.relu(tf.matmul(y11, W12) + b12)

# Print array dimensions
print("The shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())

# Calculate cross entropy and define training step
square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)

# Start tensorflow session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

# Train autoencoder
for i in range(10000):
    for j in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step1, feed_dict={x11: inputArray})
        # print("\nW11\n", sess.run(W11))
        # print("\nb11\n", sess.run(b11))
        # print("\nW12\n", sess.run(W12))
        # print("\nb12\n", sess.run(b12))

# Calculate difference between input and ouput
accuracy = tf.reduce_sum(tf.square(x11 - y12))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print(sess.run(accuracy, feed_dict={x11: inputArray}))
<commit_msg>fixed indexing<commit_after>import csv
import tensorflow as tf
import numpy as np
from src.autoencoder import autoencoder

# Read in CSV file
brca = []
count = 0

# brca is 192 long
with open("brca_toronto_collab_mutect_123_030617.csv") as csvFile:
    reader = csv.reader(csvFile)
    for row in reader:
        row.pop()
        if count != 0:
            floatRow = [float(i) for i in row]
            brca.append(floatRow)
        count += 1
csvFile.close()

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer())
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer())
y11 = tf.nn.relu(tf.matmul(x11, W11) + b11)
y12 = tf.nn.relu(tf.matmul(y11, W12) + b12)

# Print array dimensions
print("The shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())

# Calculate cross entropy and define training step
square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)

# Start tensorflow session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

# Train autoencoder
for i in range(100):
    for j in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[j], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step1, feed_dict={x11: inputArray})
        # print("\nW11\n", sess.run(W11))
        # print("\nb11\n", sess.run(b11))
        # print("\nW12\n", sess.run(W12))
        # print("\nb12\n", sess.run(b12))

# Calculate difference between input and ouput
accuracy = tf.reduce_sum(tf.square(x11 - y12))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print(sess.run(accuracy, feed_dict={x11: inputArray}))
<|endoftext|>


<commit_before>import csv
import random
import sys
import tensorflow as tf
import numpy as np
from src.autoencoder import autoencoder

# Read in CSV file
brca = []
labelInput = []
count = 0

# brca is 192 long
with open("brca_toronto_collab_mutect_123_030617.csv") as csvFile:
    reader = csv.reader(csvFile)
    for row in reader:
        row.pop()
        if count != 0:
            floatRow = [float(i) for i in row]
            brca.append(floatRow)
        count += 1
csvFile.close()

for i in range(5):
    labelInput.append([1.0, 0.0])
for i in range(12):
    labelInput.append([0.0, 1.0])
for i in range(11):
    labelInput.append([1.0, 0.0])
for i in range(95):
    labelInput.append([0.0, 1.0])

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment layer 1 vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))

z11 = tf.matmul(x11, W11) + b11
y11 = tf.nn.relu(z11)
z12 = tf.matmul(y11, W12) + b12

# Print array dimensions
print("\nThe shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())
print("The shape of y11 is: ", y11.get_shape())

# Initialize weight and adjustment layer 2 vector
W21 = tf.get_variable('W21', shape=[halfBRCA, 16], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W22 = tf.get_variable('W22', shape=[16, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
b21 = tf.Variable(tf.zeros([16]))
b22 = tf.Variable(tf.zeros([halfBRCA]))

z21 = tf.matmul(y11, W21) + b21
y21 = tf.nn.relu(z21)
z22 = tf.matmul(y21, W22) + b22

# Print array dimensions
print("\nThe shape of b21 is: ", b21.get_shape())
print("The shape of b22 is: ", b22.get_shape())
print("The shape of W21 is: ", W21.get_shape())
print("The shape of W22 is: ", W22.get_shape())
print("The shape of y21 is: ", y21.get_shape())

#Output Layer
Wo = tf.get_variable('Wo', shape=[16, 2], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
bo = tf.Variable(tf.zeros([2]))
zo = tf.matmul(y21, Wo) + bo
yo = tf.nn.softmax(zo)
labelTensor = tf.placeholder(tf.float32, [None, 2]);

print("\nThe shape of bo is: ", bo.get_shape())
print("The shape of Wo is: ", Wo.get_shape())
print("The shape of yo is: ", yo.get_shape())

# Calculate square difference
square_difference1 = tf.reduce_sum(tf.square(x11 - z12))
# square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
square_difference2 = tf.reduce_sum(tf.square(y11 - z22))
# square_difference2 = tf.reduce_sum(tf.square(y11 - y22))

# Output layer loss
loss3 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labelTensor, logits=zo))

# Optimization
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)
train_step2 = tf.train.AdamOptimizer().minimize(square_difference2)
train_step3 = tf.train.AdamOptimizer().minimize(loss3)

# Start tensorflow session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

# Train autoencoder layer 1
for j in range(1):
    for i in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
sess.run(train_step1, feed_dict={x11: inputArray})

print("\nW11\n", sess.run(W11))
print("\nb11\n", sess.run(b11))
# print("\nW12\n", sess.run(W12))
# print("\nb12\n", sess.run(b12))

# Train autoencoder layer 2
for j in range(1):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step2, feed_dict={x11: inputArray})

print("\nW21\n", sess.run(W21))
print("\nb21\n", sess.run(b21))
# print("\nW12\n", sess.run(W22))
# print("\nb12\n", sess.run(b22))

# Train autoencoder output layer
for j in range(1000):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
        # print(labelInput[i])
        labelArray = np.array(labelInput[i], dtype = float).reshape(1, 2)
        sess.run(train_step3, feed_dict={x11: inputArray, labelTensor: labelArray})

print("\nWo\n", sess.run(Wo))
print("\nbo\n", sess.run(bo))

# Print output of each layer
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    # print("\n")
    # print("y11 tensor, sample ", i, ": \n", sess.run(y11, feed_dict={x11: inputArray}))
    # print("y21 tensor, sample ", i, ": \n", sess.run(y21, feed_dict={x11: inputArray}))
    # print("yo tensor, sample ", i, ": \n", sess.run(yo, feed_dict={x11: inputArray}))

outputList = []
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    outputList.append(sess.run(yo, feed_dict={x11: inputArray}))

num1 = 0
num2 = 0
iterations = 0
for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1
    iterations += 1

print("Number of 1: ", num1)
print("Number of 2: ", num2)

# Calculate difference between input and ouput
# accuracy1 = tf.reduce_sum(tf.square(x11 - y12))
accuracy1 = tf.reduce_sum(tf.square(x11 - z12))
# accuracy2 = tf.reduce_sum(tf.square(y11 - y22))
accuracy2 = tf.reduce_sum(tf.square(y11 - z22))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print("Squared difference for layer 1: ", sess.run(accuracy1, feed_dict={x11: inputArray}))
print("Squared difference for layer 2: ", sess.run(accuracy2, feed_dict={x11: inputArray}))

<commit_msg>increased hidden layer training by 100x<commit_after>import csv
import random
import sys
import tensorflow as tf
import numpy as np
from src.autoencoder import autoencoder

# Read in CSV file
brca = []
labelInput = []
count = 0

# brca is 192 long
with open("brca_toronto_collab_mutect_123_030617.csv") as csvFile:
    reader = csv.reader(csvFile)
    for row in reader:
        row.pop()
        if count != 0:
            floatRow = [float(i) for i in row]
            brca.append(floatRow)
        count += 1
csvFile.close()

for i in range(5):
    labelInput.append([1.0, 0.0])
for i in range(12):
    labelInput.append([0.0, 1.0])
for i in range(11):
    labelInput.append([1.0, 0.0])
for i in range(95):
    labelInput.append([0.0, 1.0])

# Get the length of brca
lenBRCA = len(brca[0])
halfBRCA = lenBRCA / 2

# Initialize weight and adjustment layer 1 vectors
x11 = tf.placeholder(tf.float32, [None,lenBRCA])
b11 = tf.Variable(tf.zeros([halfBRCA]))
b12 = tf.Variable(tf.zeros([lenBRCA]))
W11 = tf.get_variable('W11', shape=[lenBRCA, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W12 = tf.get_variable('W12', shape=[halfBRCA, lenBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))

z11 = tf.matmul(x11, W11) + b11
y11 = tf.nn.relu(z11)
z12 = tf.matmul(y11, W12) + b12

# Print array dimensions
print("\nThe shape of x11 is: ", x11.get_shape())
print("The shape of b11 is: ", b11.get_shape())
print("The shape of b12 is: ", b12.get_shape())
print("The shape of W11 is: ", W11.get_shape())
print("The shape of W12 is: ", W12.get_shape())
print("The shape of y11 is: ", y11.get_shape())

# Initialize weight and adjustment layer 2 vector
W21 = tf.get_variable('W21', shape=[halfBRCA, 16], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
W22 = tf.get_variable('W22', shape=[16, halfBRCA], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
b21 = tf.Variable(tf.zeros([16]))
b22 = tf.Variable(tf.zeros([halfBRCA]))

z21 = tf.matmul(y11, W21) + b21
y21 = tf.nn.relu(z21)
z22 = tf.matmul(y21, W22) + b22

# Print array dimensions
print("\nThe shape of b21 is: ", b21.get_shape())
print("The shape of b22 is: ", b22.get_shape())
print("The shape of W21 is: ", W21.get_shape())
print("The shape of W22 is: ", W22.get_shape())
print("The shape of y21 is: ", y21.get_shape())

#Output Layer
Wo = tf.get_variable('Wo', shape=[16, 2], initializer = tf.contrib.layers.xavier_initializer(), regularizer = tf.contrib.layers.l2_regularizer(0.01))
bo = tf.Variable(tf.zeros([2]))
zo = tf.matmul(y21, Wo) + bo
yo = tf.nn.softmax(zo)
labelTensor = tf.placeholder(tf.float32, [None, 2]);

print("\nThe shape of bo is: ", bo.get_shape())
print("The shape of Wo is: ", Wo.get_shape())
print("The shape of yo is: ", yo.get_shape())

# Calculate square difference
square_difference1 = tf.reduce_sum(tf.square(x11 - z12))
# square_difference1 = tf.reduce_sum(tf.square(x11 - y12))
square_difference2 = tf.reduce_sum(tf.square(y11 - z22))
# square_difference2 = tf.reduce_sum(tf.square(y11 - y22))

# Output layer loss
loss3 = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(labels=labelTensor, logits=zo))

# Optimization
train_step1 = tf.train.AdamOptimizer().minimize(square_difference1)
train_step2 = tf.train.AdamOptimizer().minimize(square_difference2)
train_step3 = tf.train.AdamOptimizer().minimize(loss3)

# Start tensorflow session
sess = tf.InteractiveSession()
tf.global_variables_initializer().run()

# Train autoencoder layer 1
for j in range(100):
    for i in range(len(brca)):
        # Convert brca array into numpy array for tensorflow
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
sess.run(train_step1, feed_dict={x11: inputArray})

print("\nW11\n", sess.run(W11))
print("\nb11\n", sess.run(b11))
# print("\nW12\n", sess.run(W12))
# print("\nb12\n", sess.run(b12))

# Train autoencoder layer 2
for j in range(100):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype=float).reshape(1, lenBRCA)
        # print("\nInput Array\n", inputArray)
        sess.run(train_step2, feed_dict={x11: inputArray})

print("\nW21\n", sess.run(W21))
print("\nb21\n", sess.run(b21))
# print("\nW12\n", sess.run(W22))
# print("\nb12\n", sess.run(b22))

# Train autoencoder output layer
for j in range(1000):
    for i in range(len(brca)):
        inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
        # print(labelInput[i])
        labelArray = np.array(labelInput[i], dtype = float).reshape(1, 2)
        sess.run(train_step3, feed_dict={x11: inputArray, labelTensor: labelArray})

print("\nWo\n", sess.run(Wo))
print("\nbo\n", sess.run(bo))

# Print output of each layer
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    # print("\n")
    # print("y11 tensor, sample ", i, ": \n", sess.run(y11, feed_dict={x11: inputArray}))
    # print("y21 tensor, sample ", i, ": \n", sess.run(y21, feed_dict={x11: inputArray}))
    # print("yo tensor, sample ", i, ": \n", sess.run(yo, feed_dict={x11: inputArray}))

outputList = []
for i in range(len(brca)):
    inputArray = np.array(brca[i], dtype = float).reshape(1, lenBRCA)
    outputList.append(sess.run(yo, feed_dict={x11: inputArray}))

num1 = 0
num2 = 0
iterations = 0
for i in outputList:
    if i[0][0] > i[0][1]:
        num1 += 1
    else:
        num2 += 1
    iterations += 1

print("Number of 1: ", num1)
print("Number of 2: ", num2)

# Calculate difference between input and ouput
# accuracy1 = tf.reduce_sum(tf.square(x11 - y12))
accuracy1 = tf.reduce_sum(tf.square(x11 - z12))
# accuracy2 = tf.reduce_sum(tf.square(y11 - y22))
accuracy2 = tf.reduce_sum(tf.square(y11 - z22))

# Print difference
inputArray = np.array(brca[0], dtype=float).reshape(1, lenBRCA)
print("Squared difference for layer 1: ", sess.run(accuracy1, feed_dict={x11: inputArray}))
print("Squared difference for layer 2: ", sess.run(accuracy2, feed_dict={x11: inputArray}))

<|endoftext|>


<commit_before># Copyright 2009 Alejandro Sanchez <alejandro@asr-oss.com>
# Copyright 2015 Ismael Calvo <ismael.calvo@factorlibre.com>
# Copyright 2015 Tecon
# Copyright 2015 Omar Castiñeira (Comunitea)
# Copyright 2016-2020 Tecnativa - Pedro M. Baeza
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

{
    "name": "Creación de Factura-e",
    "version": "15.0.1.0.0",
    "author": "ASR-OSS, "
    "FactorLibre, "
    "Tecon, "
    "Comunitea, "
    "Tecnativa, "
    "Creu Blanca, "
    "Odoo Community Association (OCA)",
    "category": "Accounting & Finance",
    "website": "https://github.com/OCA/l10n-spain",
    "license": "AGPL-3",
    "depends": [
        "account_payment_partner",
        "l10n_es_partner",
        "l10n_es",
        "base_iso3166",
        "base_vat",
        "report_xml",
        "report_qweb_parameter",
        "l10n_es_aeat",
    ],
    "data": [
        "security/ir.model.access.csv",
        "data/account_tax_template.xml",
        "views/res_partner_view.xml",
        "views/res_company.xml",
        "views/payment_mode_view.xml",
        "views/account_tax_view.xml",
        "views/report_facturae.xml",
        "wizard/create_facturae_view.xml",
        "wizard/account_move_reversal_view.xml",
        "views/account_move_view.xml",
    ],
    "external_dependencies": {
        "python": ["cryptography", "pyOpenSSL", "pycountry", "xmlsig"]
    },
    "installable": True,
}
<commit_msg>l10n_es_facturae 15.0.1.0.1<commit_after># Copyright 2009 Alejandro Sanchez <alejandro@asr-oss.com>
# Copyright 2015 Ismael Calvo <ismael.calvo@factorlibre.com>
# Copyright 2015 Tecon
# Copyright 2015 Omar Castiñeira (Comunitea)
# Copyright 2016-2020 Tecnativa - Pedro M. Baeza
# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl.html).

{
    "name": "Creación de Factura-e",
    "version": "15.0.1.0.1",
    "author": "ASR-OSS, "
    "FactorLibre, "
    "Tecon, "
    "Comunitea, "
    "Tecnativa, "
    "Creu Blanca, "
    "Odoo Community Association (OCA)",
    "category": "Accounting & Finance",
    "website": "https://github.com/OCA/l10n-spain",
    "license": "AGPL-3",
    "depends": [
        "account_payment_partner",
        "l10n_es_partner",
        "l10n_es",
        "base_iso3166",
        "base_vat",
        "report_xml",
        "report_qweb_parameter",
        "l10n_es_aeat",
    ],
    "data": [
        "security/ir.model.access.csv",
        "data/account_tax_template.xml",
        "views/res_partner_view.xml",
        "views/res_company.xml",
        "views/payment_mode_view.xml",
        "views/account_tax_view.xml",
        "views/report_facturae.xml",
        "wizard/create_facturae_view.xml",
        "wizard/account_move_reversal_view.xml",
        "views/account_move_view.xml",
    ],
    "external_dependencies": {
        "python": ["cryptography", "pyOpenSSL", "pycountry", "xmlsig"]
    },
    "installable": True,
}
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

import functools
import json
import os
import re
import shlex
import shutil
import sys

from argparse import ArgumentParser
from itertools import product
from os import getcwd
from os.path import exists, isdir, isfile, join
from subprocess import CalledProcessError, check_call

from termcolor import colored
from yaml import safe_load

__version__ = '0.0.8'

def log(message=''):
	print(message)

def log_command(command):
	log('$ %s' % (colored(command, attrs=['bold']),))

def log_error(error):
	log(colored('%s' % (error,), 'red'))

def as_tuple(obj):
	return tuple(obj) if isinstance(obj, (list, tuple)) else (obj,)

try:
	unicode = unicode
	PY3 = False

	def native_str(s):
		if isinstance(s, unicode):
			s = s.encode('utf-8')
		assert isinstance(s, str)
		return s
except NameError:
	PY3 = True

	def native_str(s):
		assert isinstance(s, str)
		return s

def native_str_result(function):
	@functools.wraps(function)
	def wrapper(*args, **kwargs):
		result = function(*args, **kwargs)
		return native_str(result)
	return wrapper

class Structure(object):
	fields = ()

	def __hash__(self):
		return hash([getattr(self, str(f)) for f in self.fields])

	def __eq__(self, other):
		return (
			type(self) == type(other) and
			[getattr(self, str(f)) for f in self.fields] == [getattr(other, str(f)) for f in self.fields]
		)

	def __repr__(self):
		return '%s(%s)' % (
			self.__class__.__name__,
			', '.join(('%s=%r' % (key, getattr(self, str(key))) for key in self.fields))
		)

	def __unicode__(self):
		return self.__str__().decode('utf-8')

class Step(Structure):
	fields = ('name', 'commands', 'can_fail',)

	def __init__(self, name, commands, can_fail=False, check_call=check_call):
		self.name = name
		self.commands = commands
		self.can_fail = can_fail
		self.check_call = check_call

	def perform(self, environ):
		try:
			for command in self.commands:
				self.execute(command, environ)
		except Exception:
			log_error('Error performing %r step' % (self,))
			if not self.can_fail:
				raise

	def execute(self, command, environ):
		if command.startswith('sudo'):
			log(colored(
				'%r ignored because it contains sudo reference' % (command,), 'yellow'))
		else:
			log_command(command)
			self.check_call(command, shell=True, env=environ)

	@native_str_result
	def __str__(self):
		return self.name

class Build(Structure):
	fields = ('steps',)

	def __init__(self, steps):
		self.steps = steps

	def run(self, environ):
		for step in self.steps:
			step.perform(environ)

class Configuration(Structure):
	fields = ('python', 'variables', 'can_fail', 'recreate')

	def __init__(
			self,
			python, variables, base_path='.travis-solo', can_fail=False, recreate=False,
			check_call=check_call, isdir=isdir, environ=os.environ, exists=exists, rmtree=shutil.rmtree
	):
		self.base_path = base_path
		self.python = python
		self.variables = variables
		self.can_fail = can_fail
		self.recreate = recreate
		self.check_call = check_call
		self.isdir = isdir
		self.environ = environ.copy()
		self.exists = exists
		self.rmtree = rmtree

	@property
	def virtualenv_path(self):
		return join(self.base_path, re.sub(r'[\s,()]', '_', '%s' % self))

	@property
	def full_python(self):
		return ('python' if not self.python.startswith('py') else '') + self.python

	def run_build(self, build):
		original_environ = self.environ.copy()
		try:
			log(colored('\n\nBuild configuration %s running' % (self,), attrs=['bold']))
			log('Preparing the environment')
			self.prepare_virtualenv()
			self.prepare_environment()
			build.run(self.environ)
		except Exception as e:
			log_error(e)
			raise
		finally:
			self.environ.clear()
			self.environ.update(original_environ)

	def prepare_virtualenv(self):
		if self.recreate:
			self.rmtree(self.virtualenv_path, ignore_errors=True)
		elif self.exists(self.virtualenv_path):
			return

		try:
			command = 'virtualenv --distribute --python=%s %s' % (
				self.full_python, self.virtualenv_path)
			log_command(command)
			self.check_call(command, shell=True, env=self.environ)
		except OSError as e:
			log_error(e)
			raise Exception('No virtualenv executable found, please install virtualenv')
		except CalledProcessError as e:
			if e.returncode == 3:
				log_error(e)
				raise Exception('Interpreter not found')
			else:
				raise

	def prepare_environment(self):
		self.environ['PATH'] = ':'.join((join(self.virtualenv_path, 'bin'), self.environ['PATH']))

		try:
			del self.environ['__PYVENV_LAUNCHER__']
		except KeyError:
			pass

		for name in ('CI', 'TRAVIS', 'TRAVIS_SOLO'):
			self.environ[name] = 'true'

		self.environ['TRAVIS_PYTHON_VERSION'] = self.python

		self.environ.update(self.variables)

	@native_str_result
	def __str__(self):
		envs = ', '.join(('%s=%r' % (k, v) for (k, v) in self.variables.items()))
		return self.full_python + (' (%s)' % (envs,) if envs else '')

class Runner(Structure):
	fields = ('build', 'configurations')

	def __init__(self, build, configurations):
		self.build = build
		self.configurations = configurations

	def _run(self, configuration):
		try:
			configuration.run_build(self.build)
			return (configuration, True, 'Build succeeded')
		except Exception as e:
			return (configuration, False, e)

	def run(self, jobs=1):
		if jobs > 1:
			from multiprocessing.pool import ThreadPool
			pool = ThreadPool(jobs)
			results = pool.map(self._run, self.configurations)
		else:
			results = [self._run(c) for c in self.configurations]

		log(colored('\n\nBuild summary:', attrs=['bold']))
		for conf, result, message in results:
			color = 'green' if result else 'red'
			log(colored('%s: %s' % (conf, message), color))

		success = all(result for (configuration, result, _) in results if not configuration.can_fail)
		return 0 if success else 1

class Loader(object):
	def load_steps(self, settings):
		lifecycle = (
			('before_install', False),
			('install', False),
			('before_script', False),
			('script', False),
			('after_script', True),
		)

		steps = []
		for name, can_fail in lifecycle:
			commands = as_tuple(settings.get(name, []))
			if commands:
				steps.append(Step(
					name=name,
					commands=commands,
					can_fail=can_fail
				))

		return tuple(steps)

	def load_configurations(self, settings, recreate=False):
		assert settings['language'] == 'python', 'Only Python projects are supported right now'

		versions = as_tuple(settings.get('python', '2.7'))
		env_sets = [self.parse_env_set(es) for es in as_tuple(settings.get('env', ''))]

		build_matrix = list(product(versions, env_sets))

		matrix = settings.get('matrix', {})
		include = matrix.get('include', [])
		for i in include:
			version = i['python']
			env_set = self.parse_env_set(i.get('env', ''))
			element = (version, env_set)
			if element not in build_matrix:
				build_matrix.append(element)

		exclude = matrix.get('exclude', [])
		for e in exclude:
			version = e['python']
			env_set = self.parse_env_set(e.get('env', ''))
			element = (version, env_set)
			if element in build_matrix:
				build_matrix.remove(element)

		configurations = tuple((
			Configuration(python=str(p), variables=dict(v), base_path=join(getcwd(), '.travis-solo'), recreate=recreate)
			for (p, v) in build_matrix
		))
		allow_failures = matrix.get('allow_failures', [])
		for af in allow_failures:
			python = af.get('python')
			variables = dict(self.parse_env_set(af.get('env', '')))
			for c in configurations:
				if c.python == python and c.variables == variables:
					c.can_fail = True

		return configurations

	def parse_env_set(self, env_set):
		# Prior to Python 2.7.3 shlex.split didn't support unicode input
		if not PY3:
			env_set = env_set.encode('utf-8')

		assignments = shlex.split(env_set)

		if not PY3:
			assignments = [a.decode('utf-8') for a in assignments]

		return tuple(tuple(a.split('=', 1)) for a in assignments)

class Application(object):
	def __init__(self, getcwd=getcwd, isfile=isfile, open=open):
		self.getcwd = getcwd
		self.isfile = isfile
		self.open = open

	def run(self, argv):
		args = self.get_args(argv)
		settings = self.get_settings(args)

		loader = Loader()
		steps = loader.load_steps(settings)
		configurations = loader.load_configurations(settings, args.recreate)

		build = Build(steps)
		runner = Runner(build, configurations)
		sys.exit(runner.run(args.jobs))

	def get_args(self, argv):
		parser = ArgumentParser(description='Local Travis build runner')
		parser.add_argument(
			'--overwrite', dest='overwrite', action='store',
			help='Overwrite settings loaded from file with JSON-encoded dict. Usage:\n'
			     '''--overwrite '{"python": "2.7", "env": ["A=a", "A=b"]}' ''')
		parser.add_argument(
			'-j', '--jobs', type=int, default=1,
			help='number of parallel jobs; '
			     'match CPU count if value is less than 1')
		parser.add_argument(
			'--recreate', action='store_true',
			help='recreate the virtualenvs')
		parser.add_argument('--version', action='version', version=__version__)

		args = parser.parse_args(argv[1:])

		if args.jobs < 1:
			# Do not import multiprocessing globally in case it is not
			# supported on the platform.
			import multiprocessing
			args.jobs = multiprocessing.cpu_count()

		return args

	def get_settings(self, args):
		travis_file = join(self.getcwd(), '.travis.yml')
		assert self.isfile(travis_file), 'File %s not found' % (travis_file,)

		with self.open(travis_file) as fd:
			settings = safe_load(fd)

		overwrite = args.overwrite or '{}'
		decoded = json.loads(overwrite)
		settings.update(decoded)

		return settings

def main():
	app = Application()
	app.run(sys.argv)

if __name__ == '__main__':
	main()
<commit_msg>Bump version to 0.0.9<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function, unicode_literals

import functools
import json
import os
import re
import shlex
import shutil
import sys

from argparse import ArgumentParser
from itertools import product
from os import getcwd
from os.path import exists, isdir, isfile, join
from subprocess import CalledProcessError, check_call

from termcolor import colored
from yaml import safe_load

__version__ = '0.0.9'

def log(message=''):
	print(message)

def log_command(command):
	log('$ %s' % (colored(command, attrs=['bold']),))

def log_error(error):
	log(colored('%s' % (error,), 'red'))

def as_tuple(obj):
	return tuple(obj) if isinstance(obj, (list, tuple)) else (obj,)

try:
	unicode = unicode
	PY3 = False

	def native_str(s):
		if isinstance(s, unicode):
			s = s.encode('utf-8')
		assert isinstance(s, str)
		return s
except NameError:
	PY3 = True

	def native_str(s):
		assert isinstance(s, str)
		return s

def native_str_result(function):
	@functools.wraps(function)
	def wrapper(*args, **kwargs):
		result = function(*args, **kwargs)
		return native_str(result)
	return wrapper

class Structure(object):
	fields = ()

	def __hash__(self):
		return hash([getattr(self, str(f)) for f in self.fields])

	def __eq__(self, other):
		return (
			type(self) == type(other) and
			[getattr(self, str(f)) for f in self.fields] == [getattr(other, str(f)) for f in self.fields]
		)

	def __repr__(self):
		return '%s(%s)' % (
			self.__class__.__name__,
			', '.join(('%s=%r' % (key, getattr(self, str(key))) for key in self.fields))
		)

	def __unicode__(self):
		return self.__str__().decode('utf-8')

class Step(Structure):
	fields = ('name', 'commands', 'can_fail',)

	def __init__(self, name, commands, can_fail=False, check_call=check_call):
		self.name = name
		self.commands = commands
		self.can_fail = can_fail
		self.check_call = check_call

	def perform(self, environ):
		try:
			for command in self.commands:
				self.execute(command, environ)
		except Exception:
			log_error('Error performing %r step' % (self,))
			if not self.can_fail:
				raise

	def execute(self, command, environ):
		if command.startswith('sudo'):
			log(colored(
				'%r ignored because it contains sudo reference' % (command,), 'yellow'))
		else:
			log_command(command)
			self.check_call(command, shell=True, env=environ)

	@native_str_result
	def __str__(self):
		return self.name

class Build(Structure):
	fields = ('steps',)

	def __init__(self, steps):
		self.steps = steps

	def run(self, environ):
		for step in self.steps:
			step.perform(environ)

class Configuration(Structure):
	fields = ('python', 'variables', 'can_fail', 'recreate')

	def __init__(
			self,
			python, variables, base_path='.travis-solo', can_fail=False, recreate=False,
			check_call=check_call, isdir=isdir, environ=os.environ, exists=exists, rmtree=shutil.rmtree
	):
		self.base_path = base_path
		self.python = python
		self.variables = variables
		self.can_fail = can_fail
		self.recreate = recreate
		self.check_call = check_call
		self.isdir = isdir
		self.environ = environ.copy()
		self.exists = exists
		self.rmtree = rmtree

	@property
	def virtualenv_path(self):
		return join(self.base_path, re.sub(r'[\s,()]', '_', '%s' % self))

	@property
	def full_python(self):
		return ('python' if not self.python.startswith('py') else '') + self.python

	def run_build(self, build):
		original_environ = self.environ.copy()
		try:
			log(colored('\n\nBuild configuration %s running' % (self,), attrs=['bold']))
			log('Preparing the environment')
			self.prepare_virtualenv()
			self.prepare_environment()
			build.run(self.environ)
		except Exception as e:
			log_error(e)
			raise
		finally:
			self.environ.clear()
			self.environ.update(original_environ)

	def prepare_virtualenv(self):
		if self.recreate:
			self.rmtree(self.virtualenv_path, ignore_errors=True)
		elif self.exists(self.virtualenv_path):
			return

		try:
			command = 'virtualenv --distribute --python=%s %s' % (
				self.full_python, self.virtualenv_path)
			log_command(command)
			self.check_call(command, shell=True, env=self.environ)
		except OSError as e:
			log_error(e)
			raise Exception('No virtualenv executable found, please install virtualenv')
		except CalledProcessError as e:
			if e.returncode == 3:
				log_error(e)
				raise Exception('Interpreter not found')
			else:
				raise

	def prepare_environment(self):
		self.environ['PATH'] = ':'.join((join(self.virtualenv_path, 'bin'), self.environ['PATH']))

		try:
			del self.environ['__PYVENV_LAUNCHER__']
		except KeyError:
			pass

		for name in ('CI', 'TRAVIS', 'TRAVIS_SOLO'):
			self.environ[name] = 'true'

		self.environ['TRAVIS_PYTHON_VERSION'] = self.python

		self.environ.update(self.variables)

	@native_str_result
	def __str__(self):
		envs = ', '.join(('%s=%r' % (k, v) for (k, v) in self.variables.items()))
		return self.full_python + (' (%s)' % (envs,) if envs else '')

class Runner(Structure):
	fields = ('build', 'configurations')

	def __init__(self, build, configurations):
		self.build = build
		self.configurations = configurations

	def _run(self, configuration):
		try:
			configuration.run_build(self.build)
			return (configuration, True, 'Build succeeded')
		except Exception as e:
			return (configuration, False, e)

	def run(self, jobs=1):
		if jobs > 1:
			from multiprocessing.pool import ThreadPool
			pool = ThreadPool(jobs)
			results = pool.map(self._run, self.configurations)
		else:
			results = [self._run(c) for c in self.configurations]

		log(colored('\n\nBuild summary:', attrs=['bold']))
		for conf, result, message in results:
			color = 'green' if result else 'red'
			log(colored('%s: %s' % (conf, message), color))

		success = all(result for (configuration, result, _) in results if not configuration.can_fail)
		return 0 if success else 1

class Loader(object):
	def load_steps(self, settings):
		lifecycle = (
			('before_install', False),
			('install', False),
			('before_script', False),
			('script', False),
			('after_script', True),
		)

		steps = []
		for name, can_fail in lifecycle:
			commands = as_tuple(settings.get(name, []))
			if commands:
				steps.append(Step(
					name=name,
					commands=commands,
					can_fail=can_fail
				))

		return tuple(steps)

	def load_configurations(self, settings, recreate=False):
		assert settings['language'] == 'python', 'Only Python projects are supported right now'

		versions = as_tuple(settings.get('python', '2.7'))
		env_sets = [self.parse_env_set(es) for es in as_tuple(settings.get('env', ''))]

		build_matrix = list(product(versions, env_sets))

		matrix = settings.get('matrix', {})
		include = matrix.get('include', [])
		for i in include:
			version = i['python']
			env_set = self.parse_env_set(i.get('env', ''))
			element = (version, env_set)
			if element not in build_matrix:
				build_matrix.append(element)

		exclude = matrix.get('exclude', [])
		for e in exclude:
			version = e['python']
			env_set = self.parse_env_set(e.get('env', ''))
			element = (version, env_set)
			if element in build_matrix:
				build_matrix.remove(element)

		configurations = tuple((
			Configuration(python=str(p), variables=dict(v), base_path=join(getcwd(), '.travis-solo'), recreate=recreate)
			for (p, v) in build_matrix
		))
		allow_failures = matrix.get('allow_failures', [])
		for af in allow_failures:
			python = af.get('python')
			variables = dict(self.parse_env_set(af.get('env', '')))
			for c in configurations:
				if c.python == python and c.variables == variables:
					c.can_fail = True

		return configurations

	def parse_env_set(self, env_set):
		# Prior to Python 2.7.3 shlex.split didn't support unicode input
		if not PY3:
			env_set = env_set.encode('utf-8')

		assignments = shlex.split(env_set)

		if not PY3:
			assignments = [a.decode('utf-8') for a in assignments]

		return tuple(tuple(a.split('=', 1)) for a in assignments)

class Application(object):
	def __init__(self, getcwd=getcwd, isfile=isfile, open=open):
		self.getcwd = getcwd
		self.isfile = isfile
		self.open = open

	def run(self, argv):
		args = self.get_args(argv)
		settings = self.get_settings(args)

		loader = Loader()
		steps = loader.load_steps(settings)
		configurations = loader.load_configurations(settings, args.recreate)

		build = Build(steps)
		runner = Runner(build, configurations)
		sys.exit(runner.run(args.jobs))

	def get_args(self, argv):
		parser = ArgumentParser(description='Local Travis build runner')
		parser.add_argument(
			'--overwrite', dest='overwrite', action='store',
			help='Overwrite settings loaded from file with JSON-encoded dict. Usage:\n'
			     '''--overwrite '{"python": "2.7", "env": ["A=a", "A=b"]}' ''')
		parser.add_argument(
			'-j', '--jobs', type=int, default=1,
			help='number of parallel jobs; '
			     'match CPU count if value is less than 1')
		parser.add_argument(
			'--recreate', action='store_true',
			help='recreate the virtualenvs')
		parser.add_argument('--version', action='version', version=__version__)

		args = parser.parse_args(argv[1:])

		if args.jobs < 1:
			# Do not import multiprocessing globally in case it is not
			# supported on the platform.
			import multiprocessing
			args.jobs = multiprocessing.cpu_count()

		return args

	def get_settings(self, args):
		travis_file = join(self.getcwd(), '.travis.yml')
		assert self.isfile(travis_file), 'File %s not found' % (travis_file,)

		with self.open(travis_file) as fd:
			settings = safe_load(fd)

		overwrite = args.overwrite or '{}'
		decoded = json.loads(overwrite)
		settings.update(decoded)

		return settings

def main():
	app = Application()
	app.run(sys.argv)

if __name__ == '__main__':
	main()
<|endoftext|>