<commit_before>#!/usr/bin/python3

__author__ = 'm_messiah'
import logging
import signal
from requests import Session
import tornado.ioloop
import tornado.web
import tornado.escape
from base64 import b64decode

from bot_token import BOT_TOKEN

logging.basicConfig(filename="conversation.log", level=logging.INFO)

# noinspection PyAbstractClass
class Handler(tornado.web.RequestHandler):
        def post(self):
            try:
                logging.debug("Got request: %s" % self.request.body)
                update = tornado.escape.json_decode(self.request.body)
                message = update['message']
                text = message['text']

                logging.info("MESSAGE\t%s (%s)\t%s" % (
                    message['from']['username'],
                    message['from']['id'],
                    text))

                if text[0] == '/':
                    command, *arguments = text.split(" ", 1)
                    reply = CMD.get(command, not_found)(arguments, message)

                    logging.info("REPLY\t%s (%s)\t%s" % (
                        message['from']['username'],
                        message['from']['id'],
                        reply
                    ))

                    self.sendMessage(reply, command, message['from']['id'])
                else:
                    logging.info("COMMAND NOT FOUND\t%s (%s)" % (
                        message['from']['username'],
                        message['from']['id'],
                    ))
                    self.sendMessage(not_found("", message), "/not_found",
                                     message['from']['id'])
            except Exception as e:
                logging.warning(str(e))

        def sendMessage(self, text, command, id):
            payload = {
                'chat_id': id,
                'text': text,
                # 'reply_markup': keyboard
            }

            api.post(URL + "sendMessage", data=payload)

def about(arguments, message):
    return """Hey, %s!
My author is @m_messiah.
You can find this nickname at:
    + Telegram
    + Twitter
    + Instagram
    + VK
    + GitHub (m-messiah)
    """ % message["from"]["first_name"]

def not_found(arguments, message):
    return "Command not found"

def base64_decode(arguments, message):
    try:
        return b64decode(" ".join(arguments).encode("utf8"))
    except:
        return "Can't decode it"

URL = "https://api.telegram.org/bot%s/" % BOT_TOKEN
CMD = {
    "/whoisyourdaddy": about,
    "/base64": base64_decode,
}
MyURL = "https://messiah.ddns.net/telegram/"
MyURL = ""
api = Session()
application = tornado.web.Application([
    (r"/", Handler),
])

def signal_term_handler(signal, frame):
    logging.error("Got SIGTERM. Quit.")
    api.get(URL + "setWebhook?url=")
    exit(0)

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_term_handler)
    try:
        set_hook = api.get(URL + "setWebhook?url=%s" % MyURL)
        if set_hook.status_code != 200:
            logging.error("Can't set hook: %s. Quit." % set_hook.text)
            exit(1)
        application.listen(8888)
        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        signal_term_handler(signal.SIGTERM, None)
<commit_msg>/help<commit_after>#!/usr/bin/python3

__author__ = 'm_messiah'
import logging
import signal
from requests import Session
import tornado.ioloop
import tornado.web
import tornado.escape
from base64 import b64decode

from bot_token import BOT_TOKEN

logging.basicConfig(filename="conversation.log", level=logging.INFO)

# noinspection PyAbstractClass
class Handler(tornado.web.RequestHandler):
        def post(self):
            try:
                logging.debug("Got request: %s" % self.request.body)
                update = tornado.escape.json_decode(self.request.body)
                message = update['message']
                text = message['text']

                logging.info("MESSAGE\t%s (%s)\t%s" % (
                    message['from']['username'],
                    message['from']['id'],
                    text))

                if text[0] == '/':
                    command, *arguments = text.split(" ", 1)
                    reply = CMD.get(command, not_found)(arguments, message)

                    logging.info("REPLY\t%s (%s)\t%s" % (
                        message['from']['username'],
                        message['from']['id'],
                        reply
                    ))

                    self.sendMessage(reply, command, message['from']['id'])
                else:
                    logging.info("COMMAND NOT FOUND\t%s (%s)" % (
                        message['from']['username'],
                        message['from']['id'],
                    ))
                    self.sendMessage(not_found("", message), "/not_found",
                                     message['from']['id'])
            except Exception as e:
                logging.warning(str(e))

        def sendMessage(self, text, command, id):
            payload = {
                'chat_id': id,
                'text': text,
                # 'reply_markup': keyboard
            }

            api.post(URL + "sendMessage", data=payload)

def about(arguments, message):
    return """Hey, %s!
My author is @m_messiah.
You can find this nickname at:
    + Telegram
    + Twitter
    + Instagram
    + VK
    + GitHub (m-messiah)
    """ % message["from"]["first_name"]

def not_found(arguments, message):
    return "Command not found"

def base64_decode(arguments, message):
    try:
        return b64decode(" ".join(arguments).encode("utf8"))
    except:
        return "Can't decode it"

def help_message(arguments, message):
    result = "Hey, %s!\nI can accept only these commands:\n" % message["from"]["first_name"]
    for command in CMD:
        result += "\t%s\n" % command

    return result

URL = "https://api.telegram.org/bot%s/" % BOT_TOKEN
CMD = {
    "/whoisyourdaddy": about,
    "/base64": base64_decode,
    "/help": help_message,
}
MyURL = "https://messiah.ddns.net/telegram/"
MyURL = ""
api = Session()
application = tornado.web.Application([
    (r"/", Handler),
])

def signal_term_handler(signal, frame):
    logging.error("Got SIGTERM. Quit.")
    api.get(URL + "setWebhook?url=")
    exit(0)

if __name__ == '__main__':
    signal.signal(signal.SIGTERM, signal_term_handler)
    try:
        set_hook = api.get(URL + "setWebhook?url=%s" % MyURL)
        if set_hook.status_code != 200:
            logging.error("Can't set hook: %s. Quit." % set_hook.text)
            exit(1)
        application.listen(8888)
        tornado.ioloop.IOLoop.current().start()
    except KeyboardInterrupt:
        signal_term_handler(signal.SIGTERM, None)
<|endoftext|>


<commit_before># @license
# Copyright 2016 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import collections
import time
import threading

import numpy as np

from .chunks import encode_jpeg, encode_npz, encode_raw
from .token import make_random_token
from . import downsample
from . import downsample_scales

class MeshImplementationNotAvailable(Exception):
    pass

class MeshesNotSupportedForVolume(Exception):
    pass

class InvalidObjectIdForMesh(Exception):
    pass

DownsamplingScaleInfo = collections.namedtuple('DownsamplingScaleInfo', ['key',
                                                                         'downsample_factor',
                                                                         'voxel_size',
                                                                         'shape', ])

def get_scale_key(scale):
    return '%d,%d,%d' % scale

class ServedVolume(object):
    def __init__(self,
                 data,
                 offset=None,
                 voxel_offset=None,
                 voxel_size=(1, 1, 1),
                 encoding='npz',
                 max_voxels_per_chunk_log2=None,
                 volume_type=None,
                 mesh_options=None,
                 downsampling='3d',
                 max_downsampling=downsample_scales.DEFAULT_MAX_DOWNSAMPLING,
                 max_downsampled_size=downsample_scales.DEFAULT_MAX_DOWNSAMPLED_SIZE,
                 max_downsampling_scales=downsample_scales.DEFAULT_MAX_DOWNSAMPLING_SCALES):
        """Initializes a ServedVolume.

        @param data: 3-d [z, y, x] array or 4-d [channel, z, y, x] array.

        @param downsampling: '3d' to use isotropic downsampling, '2d' to downsample separately in
            XY, XZ, and YZ, None to use no downsampling.

        @param max_downsampling: Maximum amount by which on-the-fly downsampling may reduce the
            volume of a chunk.  For example, 4x4x4 downsampling reduces the volume by 64.

        @param volume_type: either 'image' or 'segmentation'.  If not specified, guessed from the
            data type.

        @param voxel_size: Sequence [x, y, z] of floats.  Specifies the voxel size.

        @param mesh_options: A dict with the following keys specifying options for mesh
            simplification for 'segmentation' volumes:

                - max_quadrics_error: float.  Edge collapses with a larger associated quadrics error
                  than this amount are prohibited.  Set this to a negative number to disable mesh
                  simplification, and just use the original mesh produced by the marching cubes
                  algorithm.  Defaults to 1e6.  The effect of this value depends on the voxel_size.

                - max_normal_angle_deviation: float.  Edge collapses that change a triangle normal
                  by more than this angle are prohibited.  The angle is specified in degrees.
                  Defaults to 90.

                - lock_boundary_vertices: bool.  Retain all vertices along mesh surface boundaries,
                  which can only occur at the boundary of the volume.  Defaults to true.
        """
        self.token = make_random_token()
        self.max_voxels_per_chunk_log2 = max_voxels_per_chunk_log2
        self.data = data
        if voxel_offset is not None:
            if offset is not None:
                raise ValueError('Must specify at most one of \'offset\' and \'voxel_offset\'.')
            offset = tuple(voxel_offset * voxel_size)
        if offset is None:
            offset = (0, 0, 0)
        self.offset = offset
        self.data_type = data.dtype.name
        if self.data_type == 'float64':
            self.data_type = 'float32'
        self.encoding = encoding
        if len(data.shape) == 3:
            self.num_channels = 1
            original_shape = data.shape[::-1]
        else:
            if len(data.shape) != 4:
                raise ValueError('data array must be 3- or 4-dimensional.')
            self.num_channels = data.shape[0]
            original_shape = data.shape[1:][::-1]
        original_shape = np.array(original_shape)
        if volume_type is None:
            if self.num_channels == 1 and (self.data_type == 'uint16' or
                                           self.data_type == 'uint32' or
                                           self.data_type == 'uint64'):
                volume_type = 'segmentation'
            else:
                volume_type = 'image'
        self.volume_type = volume_type

        self._mesh_generator = None
        self._mesh_generator_pending = False
        self._mesh_generator_lock = threading.Condition()
        self._mesh_options = mesh_options.copy() if mesh_options is not None else dict()

        voxel_size = np.array(voxel_size)
        self.voxel_size = voxel_size

        self.two_dimensional_scales = None
        self.three_dimensional_scales = None

        if downsampling is None:
            downsampling_scales = self.three_dimensional_scales = [(1, 1, 1)]
        elif downsampling == '3d':
            self.three_dimensional_scales = downsample_scales.compute_near_isotropic_downsampling_scales(
                size=original_shape,
                voxel_size=voxel_size,
                dimensions_to_downsample=[0, 1, 2],
                max_downsampling=max_downsampling,
                max_downsampled_size=max_downsampled_size,
                max_scales=max_downsampling_scales)
            downsampling_scales = self.three_dimensional_scales
        elif downsampling == '2d':
            self.two_dimensional_scales = downsample_scales.compute_two_dimensional_near_isotropic_downsampling_scales(
                size=original_shape,
                voxel_size=voxel_size,
                max_downsampling=max_downsampling,
                max_downsampled_size=max_downsampled_size,
                max_scales=max_downsampling_scales)
            downsampling_scales = [s for level in self.two_dimensional_scales for s in level]
        downsampling_scale_info = self.downsampling_scale_info = {}
        for scale in downsampling_scales:
            info = DownsamplingScaleInfo(key=get_scale_key(scale),
                                         voxel_size=tuple(voxel_size * scale),
                                         downsample_factor=scale,
                                         shape=tuple(np.cast[int](np.ceil(original_shape / scale))))
            downsampling_scale_info[info.key] = info

    def info(self):
        info = dict(volumeType=self.volume_type,
                    dataType=self.data_type,
                    encoding=self.encoding,
                    numChannels=self.num_channels)
        if self.max_voxels_per_chunk_log2 is not None:
            info['maxVoxelsPerChunkLog2'] = self.max_voxels_per_chunk_log2

        def get_scale_info(s):
            info = self.downsampling_scale_info[get_scale_key(s)]
            return dict(key='%s/%s' % (self.token, info.key),
                        offset=self.offset,
                        sizeInVoxels=info.shape,
                        voxelSize=info.voxel_size)

        if self.two_dimensional_scales is not None:
            info['twoDimensionalScales'] = [[get_scale_info(s) for s in level]
                                            for level in self.two_dimensional_scales]
        if self.three_dimensional_scales is not None:
            info['threeDimensionalScales'] = [get_scale_info(s)
                                              for s in self.three_dimensional_scales]
        return info

    def get_encoded_subvolume(self, data_format, start, end, scale_key='1,1,1'):
        scale_info = self.downsampling_scale_info.get(scale_key)
        if scale_info is None:
            raise ValueError('Invalid scale.')
        shape = scale_info.shape
        downsample_factor = scale_info.downsample_factor
        for i in xrange(3):
            if end[i] < start[i] or start[i] < 0 or end[i] > shape[i]:
                raise ValueError('Out of bounds data request.')

        indexing_expr = tuple(np.s_[start[i] * downsample_factor[i]:end[i] * downsample_factor[i]]
                              for i in (2, 1, 0))
        if len(self.data.shape) == 3:
            full_downsample_factor = downsample_factor[::-1]
            subvol = self.data[indexing_expr]
        else:
            full_downsample_factor = (1, ) + downsample_factor[::-1]
            subvol = self.data[(np.s_[:], ) + indexing_expr]
        if subvol.dtype == 'float64':
            subvol = np.cast[np.float32](subvol)

        if downsample_factor != (1, 1, 1):
            if self.volume_type == 'image':
                subvol = downsample.downsample_with_averaging(subvol, full_downsample_factor)
            else:
                subvol = downsample.downsample_with_striding(subvol, full_downsample_factor)
        content_type = 'application/octet-stream'
        if data_format == 'jpeg':
            data = encode_jpeg(subvol)
            content_type = 'image/jpeg'
        elif data_format == 'npz':
            data = encode_npz(subvol)
        elif data_format == 'raw':
            data = encode_raw(subvol)
        else:
            raise ValueError('Invalid data format requested.')
        return data, content_type

    def get_object_mesh(self, object_id):
        mesh_generator = self._get_mesh_generator()
        data = mesh_generator.get_mesh(object_id)
        if data is None:
            raise InvalidObjectIdForMesh()
        return data

    def _get_mesh_generator(self):
        if self._mesh_generator is not None:
            return self._mesh_generator
        with self._mesh_generator_lock:
            if self._mesh_generator is not None:
                return self._mesh_generator
            if self._mesh_generator_pending:
                while self._mesh_generator is None:
                    self._mesh_generator_lock.wait()
                return self._mesh_generator
            try:
                from . import _neuroglancer
            except ImportError:
                raise MeshImplementationNotAvailable()
            if not (self.num_channels == 1 and
                    (self.data_type == 'uint8' or self.data_type == 'uint16' or
                     self.data_type == 'uint32' or self.data_type == 'uint64')):
                raise MeshesNotSupportedForVolume()
            self._mesh_generator_pending = True
        if len(self.data.shape) == 4:
            data = self.data[0, :, :, :]
        else:
            data = self.data
        self._mesh_generator = _neuroglancer.OnDemandObjectMeshGenerator(
            data, self.voxel_size, self.offset, **self._mesh_options)
        with self._mesh_generator_lock:
            self._mesh_generator_pending = False
            self._mesh_generator_lock.notify_all()
        return self._mesh_generator
<commit_msg>fix(python): fix handling of volume offset in mesh generation<commit_after># @license
# Copyright 2016 Google Inc.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import, division, print_function

import collections
import time
import threading

import numpy as np

from .chunks import encode_jpeg, encode_npz, encode_raw
from .token import make_random_token
from . import downsample
from . import downsample_scales

class MeshImplementationNotAvailable(Exception):
    pass

class MeshesNotSupportedForVolume(Exception):
    pass

class InvalidObjectIdForMesh(Exception):
    pass

DownsamplingScaleInfo = collections.namedtuple('DownsamplingScaleInfo', ['key',
                                                                         'downsample_factor',
                                                                         'voxel_size',
                                                                         'shape', ])

def get_scale_key(scale):
    return '%d,%d,%d' % scale

class ServedVolume(object):
    def __init__(self,
                 data,
                 offset=None,
                 voxel_offset=None,
                 voxel_size=(1, 1, 1),
                 encoding='npz',
                 max_voxels_per_chunk_log2=None,
                 volume_type=None,
                 mesh_options=None,
                 downsampling='3d',
                 max_downsampling=downsample_scales.DEFAULT_MAX_DOWNSAMPLING,
                 max_downsampled_size=downsample_scales.DEFAULT_MAX_DOWNSAMPLED_SIZE,
                 max_downsampling_scales=downsample_scales.DEFAULT_MAX_DOWNSAMPLING_SCALES):
        """Initializes a ServedVolume.

        @param data: 3-d [z, y, x] array or 4-d [channel, z, y, x] array.

        @param downsampling: '3d' to use isotropic downsampling, '2d' to downsample separately in
            XY, XZ, and YZ, None to use no downsampling.

        @param max_downsampling: Maximum amount by which on-the-fly downsampling may reduce the
            volume of a chunk.  For example, 4x4x4 downsampling reduces the volume by 64.

        @param volume_type: either 'image' or 'segmentation'.  If not specified, guessed from the
            data type.

        @param voxel_size: Sequence [x, y, z] of floats.  Specifies the voxel size.

        @param mesh_options: A dict with the following keys specifying options for mesh
            simplification for 'segmentation' volumes:

                - max_quadrics_error: float.  Edge collapses with a larger associated quadrics error
                  than this amount are prohibited.  Set this to a negative number to disable mesh
                  simplification, and just use the original mesh produced by the marching cubes
                  algorithm.  Defaults to 1e6.  The effect of this value depends on the voxel_size.

                - max_normal_angle_deviation: float.  Edge collapses that change a triangle normal
                  by more than this angle are prohibited.  The angle is specified in degrees.
                  Defaults to 90.

                - lock_boundary_vertices: bool.  Retain all vertices along mesh surface boundaries,
                  which can only occur at the boundary of the volume.  Defaults to true.
        """
        self.token = make_random_token()
        self.max_voxels_per_chunk_log2 = max_voxels_per_chunk_log2
        self.data = data
        if voxel_offset is not None:
            if offset is not None:
                raise ValueError('Must specify at most one of \'offset\' and \'voxel_offset\'.')
            offset = tuple(voxel_offset * voxel_size)
        if offset is None:
            offset = (0, 0, 0)
        self.offset = offset
        self.data_type = data.dtype.name
        if self.data_type == 'float64':
            self.data_type = 'float32'
        self.encoding = encoding
        if len(data.shape) == 3:
            self.num_channels = 1
            original_shape = data.shape[::-1]
        else:
            if len(data.shape) != 4:
                raise ValueError('data array must be 3- or 4-dimensional.')
            self.num_channels = data.shape[0]
            original_shape = data.shape[1:][::-1]
        original_shape = np.array(original_shape)
        if volume_type is None:
            if self.num_channels == 1 and (self.data_type == 'uint16' or
                                           self.data_type == 'uint32' or
                                           self.data_type == 'uint64'):
                volume_type = 'segmentation'
            else:
                volume_type = 'image'
        self.volume_type = volume_type

        self._mesh_generator = None
        self._mesh_generator_pending = False
        self._mesh_generator_lock = threading.Condition()
        self._mesh_options = mesh_options.copy() if mesh_options is not None else dict()

        voxel_size = np.array(voxel_size)
        self.voxel_size = voxel_size

        self.two_dimensional_scales = None
        self.three_dimensional_scales = None

        if downsampling is None:
            downsampling_scales = self.three_dimensional_scales = [(1, 1, 1)]
        elif downsampling == '3d':
            self.three_dimensional_scales = downsample_scales.compute_near_isotropic_downsampling_scales(
                size=original_shape,
                voxel_size=voxel_size,
                dimensions_to_downsample=[0, 1, 2],
                max_downsampling=max_downsampling,
                max_downsampled_size=max_downsampled_size,
                max_scales=max_downsampling_scales)
            downsampling_scales = self.three_dimensional_scales
        elif downsampling == '2d':
            self.two_dimensional_scales = downsample_scales.compute_two_dimensional_near_isotropic_downsampling_scales(
                size=original_shape,
                voxel_size=voxel_size,
                max_downsampling=max_downsampling,
                max_downsampled_size=max_downsampled_size,
                max_scales=max_downsampling_scales)
            downsampling_scales = [s for level in self.two_dimensional_scales for s in level]
        downsampling_scale_info = self.downsampling_scale_info = {}
        for scale in downsampling_scales:
            info = DownsamplingScaleInfo(key=get_scale_key(scale),
                                         voxel_size=tuple(voxel_size * scale),
                                         downsample_factor=scale,
                                         shape=tuple(np.cast[int](np.ceil(original_shape / scale))))
            downsampling_scale_info[info.key] = info

    def info(self):
        info = dict(volumeType=self.volume_type,
                    dataType=self.data_type,
                    encoding=self.encoding,
                    numChannels=self.num_channels)
        if self.max_voxels_per_chunk_log2 is not None:
            info['maxVoxelsPerChunkLog2'] = self.max_voxels_per_chunk_log2

        def get_scale_info(s):
            info = self.downsampling_scale_info[get_scale_key(s)]
            return dict(key='%s/%s' % (self.token, info.key),
                        offset=self.offset,
                        sizeInVoxels=info.shape,
                        voxelSize=info.voxel_size)

        if self.two_dimensional_scales is not None:
            info['twoDimensionalScales'] = [[get_scale_info(s) for s in level]
                                            for level in self.two_dimensional_scales]
        if self.three_dimensional_scales is not None:
            info['threeDimensionalScales'] = [get_scale_info(s)
                                              for s in self.three_dimensional_scales]
        return info

    def get_encoded_subvolume(self, data_format, start, end, scale_key='1,1,1'):
        scale_info = self.downsampling_scale_info.get(scale_key)
        if scale_info is None:
            raise ValueError('Invalid scale.')
        shape = scale_info.shape
        downsample_factor = scale_info.downsample_factor
        for i in xrange(3):
            if end[i] < start[i] or start[i] < 0 or end[i] > shape[i]:
                raise ValueError('Out of bounds data request.')

        indexing_expr = tuple(np.s_[start[i] * downsample_factor[i]:end[i] * downsample_factor[i]]
                              for i in (2, 1, 0))
        if len(self.data.shape) == 3:
            full_downsample_factor = downsample_factor[::-1]
            subvol = self.data[indexing_expr]
        else:
            full_downsample_factor = (1, ) + downsample_factor[::-1]
            subvol = self.data[(np.s_[:], ) + indexing_expr]
        if subvol.dtype == 'float64':
            subvol = np.cast[np.float32](subvol)

        if downsample_factor != (1, 1, 1):
            if self.volume_type == 'image':
                subvol = downsample.downsample_with_averaging(subvol, full_downsample_factor)
            else:
                subvol = downsample.downsample_with_striding(subvol, full_downsample_factor)
        content_type = 'application/octet-stream'
        if data_format == 'jpeg':
            data = encode_jpeg(subvol)
            content_type = 'image/jpeg'
        elif data_format == 'npz':
            data = encode_npz(subvol)
        elif data_format == 'raw':
            data = encode_raw(subvol)
        else:
            raise ValueError('Invalid data format requested.')
        return data, content_type

    def get_object_mesh(self, object_id):
        mesh_generator = self._get_mesh_generator()
        data = mesh_generator.get_mesh(object_id)
        if data is None:
            raise InvalidObjectIdForMesh()
        return data

    def _get_mesh_generator(self):
        if self._mesh_generator is not None:
            return self._mesh_generator
        with self._mesh_generator_lock:
            if self._mesh_generator is not None:
                return self._mesh_generator
            if self._mesh_generator_pending:
                while self._mesh_generator is None:
                    self._mesh_generator_lock.wait()
                return self._mesh_generator
            try:
                from . import _neuroglancer
            except ImportError:
                raise MeshImplementationNotAvailable()
            if not (self.num_channels == 1 and
                    (self.data_type == 'uint8' or self.data_type == 'uint16' or
                     self.data_type == 'uint32' or self.data_type == 'uint64')):
                raise MeshesNotSupportedForVolume()
            self._mesh_generator_pending = True
        if len(self.data.shape) == 4:
            data = self.data[0, :, :, :]
        else:
            data = self.data
        self._mesh_generator = _neuroglancer.OnDemandObjectMeshGenerator(
            data, self.voxel_size, self.offset / self.voxel_size, **self._mesh_options)
        with self._mesh_generator_lock:
            self._mesh_generator_pending = False
            self._mesh_generator_lock.notify_all()
        return self._mesh_generator
<|endoftext|>


<commit_before># Copyright 2014 Facebook, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither the name Facebook nor the names of its contributors may be used to
#    endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os, errno, socket, subprocess
import bser

class Unavailable(Exception):
    pass

class client(object):
    sock = None
    sockpath = None

    def __init__(self, sockpath = None, timeout = 1.0):
        self.sockpath = sockpath
        self.timeout = timeout

    def _resolvesockname(self):
        # if invoked via a trigger, watchman will set this env var; we
        # should use it unless explicitly set otherwise
        path = os.getenv('WATCHMAN_SOCK')
        if path:
            return path

        cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, close_fds=True)
        except OSError:
            raise Unavailable('"watchman" executable not in PATH')

        stdout, stderr = p.communicate()
        exitcode = p.poll()

        if exitcode:
            raise Unavailable("watchman exited with code %d" % exitcode)

        result = bser.loads(stdout)
        if 'error' in result:
            raise Unavailable('get-sockname error: %s' % result['error'])

        return result['sockname']

    def _connect(self):
        if self.sock:
            return self.sock

        if self.sockpath is None:
            self.sockpath = self._resolvesockname()

        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        try:
            sock.settimeout(self.timeout)
            sock.connect(self.sockpath)
            self.sock = sock
            return sock
        except socket.error, e:
            raise Unavailable('unable to connect to %s: %s' % (self.sockpath, e))

    def query(self, *args):
        cmd = bser.dumps(args)
        response = None
        result = None

        sock = self._connect()

        try:
            sock.sendall(cmd)
            buf = [sock.recv(8192)]
            if not buf[0]:
                raise Unavailable('empty watchman response')

            elen = bser.pdu_len(buf[0])
            rlen = len(buf[0])

            while elen > rlen:
                buf.append(sock.recv(elen - rlen))
                rlen += len(buf[-1])

            response = ''.join(buf)
        except socket.timeout:
            raise Unavailable('timed out waiting for response')

        try:
            result = bser.loads(response)
        except ValueError, e:
            raise Unavailable('watchman response decode error: %s' % e)

        if 'error' in result:
            raise Unavailable('watchman command error: %s' % result['error'])

        return result

<commit_msg>read minimum length when determining pdu length<commit_after># Copyright 2014 Facebook, Inc.
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#  * Redistributions of source code must retain the above copyright notice,
#    this list of conditions and the following disclaimer.
#
#  * Redistributions in binary form must reproduce the above copyright notice,
#    this list of conditions and the following disclaimer in the documentation
#    and/or other materials provided with the distribution.
#
#  * Neither the name Facebook nor the names of its contributors may be used to
#    endorse or promote products derived from this software without specific
#    prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

import os, errno, socket, subprocess
import bser

# 2 bytes marker, 1 byte int size, 8 bytes int64 value
sniff_len = 13

class Unavailable(Exception):
    pass

class client(object):
    sock = None
    sockpath = None

    def __init__(self, sockpath = None, timeout = 1.0):
        self.sockpath = sockpath
        self.timeout = timeout

    def _resolvesockname(self):
        # if invoked via a trigger, watchman will set this env var; we
        # should use it unless explicitly set otherwise
        path = os.getenv('WATCHMAN_SOCK')
        if path:
            return path

        cmd = ['watchman', '--output-encoding=bser', 'get-sockname']
        try:
            p = subprocess.Popen(cmd, stdout=subprocess.PIPE,
                                 stderr=subprocess.PIPE, close_fds=True)
        except OSError:
            raise Unavailable('"watchman" executable not in PATH')

        stdout, stderr = p.communicate()
        exitcode = p.poll()

        if exitcode:
            raise Unavailable("watchman exited with code %d" % exitcode)

        result = bser.loads(stdout)
        if 'error' in result:
            raise Unavailable('get-sockname error: %s' % result['error'])

        return result['sockname']

    def _connect(self):
        if self.sock:
            return self.sock

        if self.sockpath is None:
            self.sockpath = self._resolvesockname()

        sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
        try:
            sock.settimeout(self.timeout)
            sock.connect(self.sockpath)
            self.sock = sock
            return sock
        except socket.error, e:
            raise Unavailable('unable to connect to %s: %s' % (self.sockpath, e))

    def query(self, *args):
        cmd = bser.dumps(args)
        response = None
        result = None

        sock = self._connect()

        try:
            sock.sendall(cmd)
            buf = [sock.recv(sniff_len)]
            if not buf[0]:
                raise Unavailable('empty watchman response')

            elen = bser.pdu_len(buf[0])
            rlen = len(buf[0])

            while elen > rlen:
                buf.append(sock.recv(elen - rlen))
                rlen += len(buf[-1])

            response = ''.join(buf)
        except socket.timeout:
            raise Unavailable('timed out waiting for response')

        try:
            result = bser.loads(response)
        except ValueError, e:
            raise Unavailable('watchman response decode error: %s' % e)

        if 'error' in result:
            raise Unavailable('watchman command error: %s' % result['error'])

        return result

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

<commit_msg>added comment in empty exp. so python <3 (def macosx) can easily be fixed, just remove utf-8 declaration :)<commit_after># -*- coding: utf-8 -*-
# Note, on python <3, utf-8 declaration might yield exception, in that case, remove this comment
<|endoftext|>


<commit_before>import sys
import requests
from lxml.html import parse
from link_checker import check_page

def corrs_base_data():
    return [
        {
            "dataset": "HC_M2_0606_P",
            "trait_id": "1435464_at",
            "corr_dataset": "HC_M2_0606_P",
        },
        {
            "dataset": "HC_M2_0606_P",
            "trait_id": "1457545_at",
            "corr_dataset": "HC_M2_0606_R",
        },
    ]
def sample_vals():
    return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}'

def do_request(url, data):
    return requests.post(
        url,
        data={
            "dataset": "HC_M2_0606_P",
            "trait_id": "1435464_at",
            "corr_dataset": "HC_M2_0606_P",
            "corr_sample_method": "pearson",
            "corr_return_results": "100",
            "corr_samples_group": "samples_primary",
            "sample_vals": sample_vals(),
            **data,
        })

def check_sample_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "sample",
        "corr_sample_method": "pearson",
        "location_type": "gene",
        "corr_return_results": "200"
    }
    top_n_message = "The top 200 correlations ranked by the Genetic Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_tissue_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "tissue",
        "location_type": "gene",
    }
    top_n_message = "The top 100 correlations ranked by the Tissue Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_lit_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "lit",
        "corr_return_results": "200"
    }
    top_n_message = "The top 200 correlations ranked by the Literature Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find("Values of record 1435464_at") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_correlations(args_obj, parser):
    print("")
    print("Checking the correlations...")
    corr_type_fns = {
        "sample": check_sample_correlations,
        "tissue": check_tissue_correlations,
        "lit": check_lit_correlations
    }
    host = args_obj.host
    failure = False
    for corr_type, corr_type_fn in corr_type_fns.items():
        for corr_base in corrs_base_data():
            try:
                print(f"\tChecking {corr_type} correlations...", end="")
                corr_type_fn(host, corr_base)
                print(" ok")
            except AssertionError as asserterr:
                print (f" fail: {asserterr.args[0]}")
                failure = True

    if failure:
        print("FAIL!")
        sys.exit(1)
    print("OK")
<commit_msg>Fix asserts: Use variable for assert<commit_after>import sys
import requests
from lxml.html import parse
from link_checker import check_page

def corrs_base_data():
    return [
        {
            "dataset": "HC_M2_0606_P",
            "trait_id": "1435464_at",
            "corr_dataset": "HC_M2_0606_P",
        },
        {
            "dataset": "HC_M2_0606_P",
            "trait_id": "1457545_at",
            "corr_dataset": "HC_M2_0606_R",
        },
    ]
def sample_vals():
    return '{"C57BL/6J":"10.835","DBA/2J":"11.142","B6D2F1":"11.126","D2B6F1":"11.143","BXD1":"10.811","BXD2":"11.503","BXD5":"10.766","BXD6":"10.986","BXD8":"11.050","BXD9":"10.822","BXD11":"10.670","BXD12":"10.946","BXD13":"10.890","BXD14":"x","BXD15":"10.884","BXD16":"11.222","BXD18":"x","BXD19":"10.968","BXD20":"10.962","BXD21":"10.906","BXD22":"11.080","BXD23":"11.046","BXD24":"11.146","BXD24a":"x","BXD25":"x","BXD27":"11.078","BXD28":"11.034","BXD29":"10.808","BXD30":"x","BXD31":"11.087","BXD32":"11.029","BXD33":"10.662","BXD34":"11.482","BXD35":"x","BXD36":"x","BXD37":"x","BXD38":"10.836","BXD39":"10.926","BXD40":"10.638","BXD41":"x","BXD42":"10.974","BXD43":"10.828","BXD44":"10.900","BXD45":"11.358","BXD48":"11.042","BXD48a":"10.975","BXD49":"x","BXD50":"11.228","BXD51":"11.126","BXD52":"x","BXD53":"x","BXD54":"x","BXD55":"11.580","BXD56":"x","BXD59":"x","BXD60":"10.829","BXD61":"11.152","BXD62":"11.156","BXD63":"10.942","BXD64":"10.506","BXD65":"11.126","BXD65a":"11.272","BXD65b":"11.157","BXD66":"11.071","BXD67":"11.080","BXD68":"10.997","BXD69":"11.096","BXD70":"11.152","BXD71":"x","BXD72":"x","BXD73":"11.262","BXD73a":"11.444","BXD73b":"x","BXD74":"10.974","BXD75":"11.150","BXD76":"10.920","BXD77":"10.928","BXD78":"x","BXD79":"11.371","BXD81":"x","BXD83":"10.946","BXD84":"11.181","BXD85":"10.992","BXD86":"10.770","BXD87":"11.200","BXD88":"x","BXD89":"10.930","BXD90":"11.183","BXD91":"x","BXD93":"11.056","BXD94":"10.737","BXD95":"x","BXD98":"10.986","BXD99":"10.892","BXD100":"x","BXD101":"x","BXD102":"x","BXD104":"x","BXD105":"x","BXD106":"x","BXD107":"x","BXD108":"x","BXD109":"x","BXD110":"x","BXD111":"x","BXD112":"x","BXD113":"x","BXD114":"x","BXD115":"x","BXD116":"x","BXD117":"x","BXD119":"x","BXD120":"x","BXD121":"x","BXD122":"x","BXD123":"x","BXD124":"x","BXD125":"x","BXD126":"x","BXD127":"x","BXD128":"x","BXD128a":"x","BXD130":"x","BXD131":"x","BXD132":"x","BXD133":"x","BXD134":"x","BXD135":"x","BXD136":"x","BXD137":"x","BXD138":"x","BXD139":"x","BXD141":"x","BXD142":"x","BXD144":"x","BXD145":"x","BXD146":"x","BXD147":"x","BXD148":"x","BXD149":"x","BXD150":"x","BXD151":"x","BXD152":"x","BXD153":"x","BXD154":"x","BXD155":"x","BXD156":"x","BXD157":"x","BXD160":"x","BXD161":"x","BXD162":"x","BXD165":"x","BXD168":"x","BXD169":"x","BXD170":"x","BXD171":"x","BXD172":"x","BXD173":"x","BXD174":"x","BXD175":"x","BXD176":"x","BXD177":"x","BXD178":"x","BXD180":"x","BXD181":"x","BXD183":"x","BXD184":"x","BXD186":"x","BXD187":"x","BXD188":"x","BXD189":"x","BXD190":"x","BXD191":"x","BXD192":"x","BXD193":"x","BXD194":"x","BXD195":"x","BXD196":"x","BXD197":"x","BXD198":"x","BXD199":"x","BXD200":"x","BXD201":"x","BXD202":"x","BXD203":"x","BXD204":"x","BXD205":"x","BXD206":"x","BXD207":"x","BXD208":"x","BXD209":"x","BXD210":"x","BXD211":"x","BXD212":"x","BXD213":"x","BXD214":"x","BXD215":"x","BXD216":"x","BXD217":"x","BXD218":"x","BXD219":"x","BXD220":"x"}'

def do_request(url, data):
    return requests.post(
        url,
        data={
            "dataset": "HC_M2_0606_P",
            "trait_id": "1435464_at",
            "corr_dataset": "HC_M2_0606_P",
            "corr_sample_method": "pearson",
            "corr_return_results": "100",
            "corr_samples_group": "samples_primary",
            "sample_vals": sample_vals(),
            **data,
        })

def check_sample_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "sample",
        "corr_sample_method": "pearson",
        "location_type": "gene",
        "corr_return_results": "200"
    }
    top_n_message = "The top 200 correlations ranked by the Genetic Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find(f"Values of record {base_data['trait_id']}") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_tissue_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "tissue",
        "location_type": "gene",
    }
    top_n_message = "The top 100 correlations ranked by the Tissue Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find(f"Values of record {base_data['trait_id']}") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_lit_correlations(baseurl, base_data):
    data = {
        **base_data,
        "corr_type": "lit",
        "corr_return_results": "200"
    }
    top_n_message = "The top 200 correlations ranked by the Literature Correlation"
    result = do_request(f"{baseurl}/corr_compute", data)
    assert result.status_code == 200
    assert (result.text.find(f"Values of record {base_data['trait_id']}") >= 0), result.text
    assert (result.text.find(top_n_message) >= 0), result.text

def check_correlations(args_obj, parser):
    print("")
    print("Checking the correlations...")
    corr_type_fns = {
        "sample": check_sample_correlations,
        "tissue": check_tissue_correlations,
        "lit": check_lit_correlations
    }
    host = args_obj.host
    failure = False
    for corr_type, corr_type_fn in corr_type_fns.items():
        for corr_base in corrs_base_data():
            try:
                print(f"\tChecking {corr_type} correlations...", end="")
                corr_type_fn(host, corr_base)
                print(" ok")
            except AssertionError as asserterr:
                print (f" fail: {asserterr.args[0]}")
                failure = True

    if failure:
        print("FAIL!")
        sys.exit(1)
    print("OK")
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""Pythonization of the :ref:`tmux(1)` pane.

libtmux.pane
~~~~~~~~~~~~

"""
from __future__ import absolute_import, unicode_literals, with_statement

import logging

from . import exc
from .common import TmuxMappingObject, TmuxRelationalObject

logger = logging.getLogger(__name__)

class Pane(TmuxMappingObject, TmuxRelationalObject):
    """
    A :term:`tmux(1)` :term:`pane` [#]_.

    ``Pane`` instances can send commands directly to a pane, or traverse
    between linked tmux objects.

    Parameters
    ----------
    window : :class:`Window`

    Notes
    -----

    .. versionchanged:: 0.8
        Renamed from ``.tmux`` to ``.cmd``.

    References
    ----------

    .. [#] tmux pane. openbsd manpage for TMUX(1).
           "Each window displayed by tmux may be split into one or more
           panes; each pane takes up a certain area of the display and is
           a separate terminal."

       https://man.openbsd.org/tmux.1#WINDOWS_AND_PANES.
       Accessed April 1st, 2018.
    """
    #: namespace used :class:`~libtmux.common.TmuxMappingObject`
    formatter_prefix = 'pane_'

    def __init__(self, window=None, **kwargs):
        if not window:
            raise ValueError('Pane must have ``Window`` object')

        self.window = window
        self.session = self.window.session
        self.server = self.session.server

        self._pane_id = kwargs['pane_id']

        self.server._update_panes()

    @property
    def _info(self, *args):

        attrs = {
            'pane_id': self._pane_id
        }

        # from https://github.com/serkanyersen/underscore.py
        def by(val, *args):
            for key, value in attrs.items():
                try:
                    if attrs[key] != val[key]:
                        return False
                except KeyError:
                    return False
                return True

        return list(filter(by, self.server._panes))[0]

    def cmd(self, cmd, *args, **kwargs):
        """Return :meth:`Server.cmd` defaulting to ``target_pane`` as target.

        Send command to tmux with :attr:`pane_id` as ``target-pane``.

        Specifying ``('-t', 'custom-target')`` or ``('-tcustom_target')`` in
        ``args`` will override using the object's ``pane_id`` as target.

        Returns
        -------
        :class:`Server.cmd`
        """
        if not any(arg.startswith('-t') for arg in args):
            args = ('-t', self.get('pane_id')) + args

        return self.server.cmd(cmd, *args, **kwargs)

    def send_keys(self, cmd, enter=True, suppress_history=True):
        """
        ``$ tmux send-keys`` to the pane.

        A leading space character is added to cmd to avoid polluting the
        user's history.

        Parameters
        ----------
        cmd : str
            Text or input into pane
        enter : bool, optional
            Send enter after sending the input, default True.
        suppress_history : bool, optional
            Don't add these keys to the shell history, default True.
        """
        prefix = ' ' if suppress_history else ''
        self.cmd('send-keys', prefix + cmd)

        if enter:
            self.enter()

    def clear(self):
        """Clear pane."""
        self.send_keys('reset')

    def reset(self):
        """Reset and clear pane history. """

        self.cmd('send-keys', '-R \; clear-history')

    def split_window(self, attach=False, vertical=True, start_directory=None):
        """
        Split window at pane and return newly created :class:`Pane`.

        Parameters
        ----------
        attach : bool, optional
            Attach / select pane after creation.
        vertical : bool, optional
            split vertically
        start_directory : str, optional
            specifies the working directory in which the new pane is created.

        Returns
        -------
        :class:`Pane`
        """
        return self.window.split_window(
            target=self.get('pane_id'),
            start_directory=start_directory,
            attach=attach,
            vertical=vertical
        )

    def set_width(self, width):
        """
        Set width of pane.

        Parameters
        ----------
        width : int
            pane width, in cells
        """
        self.resize_pane(width=width)

    def set_height(self, height):
        """
        Set height of pane.

        Parameters
        ----------
        height : int
            height of pain, in cells
        """
        self.resize_pane(height=height)

    def resize_pane(self, *args, **kwargs):
        """
        ``$ tmux resize-pane`` of pane and return ``self``.

        Parameters
        ----------
        target_pane : str
            ``target_pane``, or ``-U``,``-D``, ``-L``, ``-R``.

        Other Parameters
        ----------------
        height : int
            ``resize-pane -y`` dimensions
        width : int
            ``resize-pane -x`` dimensions

        Returns
        -------
        :class:`Pane`

        Raises
        ------
        exc.LibTmuxException
        """

        if 'height' in kwargs:
            proc = self.cmd('resize-pane', '-y%s' % int(kwargs['height']))
        elif 'width' in kwargs:
            proc = self.cmd('resize-pane', '-x%s' % int(kwargs['width']))
        else:
            proc = self.cmd('resize-pane', args[0])

        if proc.stderr:
            raise exc.LibTmuxException(proc.stderr)

        self.server._update_panes()
        return self

    def enter(self):
        """
        Send carriage return to pane.

        ``$ tmux send-keys`` send Enter to the pane.
        """
        self.cmd('send-keys', 'Enter')

    def select_pane(self):
        """
        Select pane. Return ``self``.

        To select a window object asynchrously. If a ``pane`` object exists
        and is no longer longer the current window, ``w.select_pane()``
        will make ``p`` the current pane.

        Returns
        -------
        :class:`pane`
        """
        return self.window.select_pane(self.get('pane_id'))

    def __repr__(self):
        return "%s(%s %s)" % (
            self.__class__.__name__,
            self.get('pane_id'),
            self.window
        )
<commit_msg>Add method display_message to Pane<commit_after># -*- coding: utf-8 -*-
"""Pythonization of the :ref:`tmux(1)` pane.

libtmux.pane
~~~~~~~~~~~~

"""
from __future__ import absolute_import, unicode_literals, with_statement

import logging

from . import exc
from .common import TmuxMappingObject, TmuxRelationalObject

logger = logging.getLogger(__name__)

class Pane(TmuxMappingObject, TmuxRelationalObject):
    """
    A :term:`tmux(1)` :term:`pane` [#]_.

    ``Pane`` instances can send commands directly to a pane, or traverse
    between linked tmux objects.

    Parameters
    ----------
    window : :class:`Window`

    Notes
    -----

    .. versionchanged:: 0.8
        Renamed from ``.tmux`` to ``.cmd``.

    References
    ----------

    .. [#] tmux pane. openbsd manpage for TMUX(1).
           "Each window displayed by tmux may be split into one or more
           panes; each pane takes up a certain area of the display and is
           a separate terminal."

       https://man.openbsd.org/tmux.1#WINDOWS_AND_PANES.
       Accessed April 1st, 2018.
    """
    #: namespace used :class:`~libtmux.common.TmuxMappingObject`
    formatter_prefix = 'pane_'

    def __init__(self, window=None, **kwargs):
        if not window:
            raise ValueError('Pane must have ``Window`` object')

        self.window = window
        self.session = self.window.session
        self.server = self.session.server

        self._pane_id = kwargs['pane_id']

        self.server._update_panes()

    @property
    def _info(self, *args):

        attrs = {
            'pane_id': self._pane_id
        }

        # from https://github.com/serkanyersen/underscore.py
        def by(val, *args):
            for key, value in attrs.items():
                try:
                    if attrs[key] != val[key]:
                        return False
                except KeyError:
                    return False
                return True

        return list(filter(by, self.server._panes))[0]

    def cmd(self, cmd, *args, **kwargs):
        """Return :meth:`Server.cmd` defaulting to ``target_pane`` as target.

        Send command to tmux with :attr:`pane_id` as ``target-pane``.

        Specifying ``('-t', 'custom-target')`` or ``('-tcustom_target')`` in
        ``args`` will override using the object's ``pane_id`` as target.

        Returns
        -------
        :class:`Server.cmd`
        """
        if not any(arg.startswith('-t') for arg in args):
            args = ('-t', self.get('pane_id')) + args

        return self.server.cmd(cmd, *args, **kwargs)

    def send_keys(self, cmd, enter=True, suppress_history=True):
        """
        ``$ tmux send-keys`` to the pane.

        A leading space character is added to cmd to avoid polluting the
        user's history.

        Parameters
        ----------
        cmd : str
            Text or input into pane
        enter : bool, optional
            Send enter after sending the input, default True.
        suppress_history : bool, optional
            Don't add these keys to the shell history, default True.
        """
        prefix = ' ' if suppress_history else ''
        self.cmd('send-keys', prefix + cmd)

        if enter:
            self.enter()

    def display_message(self, cmd, get_text=False):
        """
        ``$ tmux display-message`` to the pane.

        Displays a message in target-client status line.

        Parameters
        ----------
        cmd : str
            Special parameters to request from pane.
        get_text : bool, optional
            Returns only text without displaying a message in
            target-client status line.

        Returns
        -------
        :class:`list`
        :class:`None`
        """
        if get_text:
            return self.cmd('display-message', '-p', cmd).stdout
        else:
            self.cmd('display-message', cmd)

    def clear(self):
        """Clear pane."""
        self.send_keys('reset')

    def reset(self):
        """Reset and clear pane history. """

        self.cmd('send-keys', '-R \; clear-history')

    def split_window(self, attach=False, vertical=True, start_directory=None):
        """
        Split window at pane and return newly created :class:`Pane`.

        Parameters
        ----------
        attach : bool, optional
            Attach / select pane after creation.
        vertical : bool, optional
            split vertically
        start_directory : str, optional
            specifies the working directory in which the new pane is created.

        Returns
        -------
        :class:`Pane`
        """
        return self.window.split_window(
            target=self.get('pane_id'),
            start_directory=start_directory,
            attach=attach,
            vertical=vertical
        )

    def set_width(self, width):
        """
        Set width of pane.

        Parameters
        ----------
        width : int
            pane width, in cells
        """
        self.resize_pane(width=width)

    def set_height(self, height):
        """
        Set height of pane.

        Parameters
        ----------
        height : int
            height of pain, in cells
        """
        self.resize_pane(height=height)

    def resize_pane(self, *args, **kwargs):
        """
        ``$ tmux resize-pane`` of pane and return ``self``.

        Parameters
        ----------
        target_pane : str
            ``target_pane``, or ``-U``,``-D``, ``-L``, ``-R``.

        Other Parameters
        ----------------
        height : int
            ``resize-pane -y`` dimensions
        width : int
            ``resize-pane -x`` dimensions

        Returns
        -------
        :class:`Pane`

        Raises
        ------
        exc.LibTmuxException
        """

        if 'height' in kwargs:
            proc = self.cmd('resize-pane', '-y%s' % int(kwargs['height']))
        elif 'width' in kwargs:
            proc = self.cmd('resize-pane', '-x%s' % int(kwargs['width']))
        else:
            proc = self.cmd('resize-pane', args[0])

        if proc.stderr:
            raise exc.LibTmuxException(proc.stderr)

        self.server._update_panes()
        return self

    def enter(self):
        """
        Send carriage return to pane.

        ``$ tmux send-keys`` send Enter to the pane.
        """
        self.cmd('send-keys', 'Enter')

    def select_pane(self):
        """
        Select pane. Return ``self``.

        To select a window object asynchrously. If a ``pane`` object exists
        and is no longer longer the current window, ``w.select_pane()``
        will make ``p`` the current pane.

        Returns
        -------
        :class:`pane`
        """
        return self.window.select_pane(self.get('pane_id'))

    def __repr__(self):
        return "%s(%s %s)" % (
            self.__class__.__name__,
            self.get('pane_id'),
            self.window
        )
<|endoftext|>


<commit_before>from collections import defaultdict
import shutil

from defaults import *
from vcf_utils import *
from sv_interval import SVInterval, get_gaps_file, interval_overlaps_interval_list, merge_intervals
from pindel_reader import PindelReader
from breakdancer_reader import BreakDancerReader
from breakseq_reader import BreakSeqReader
from cnvnator_reader import CNVnatorReader
from generate_sv_intervals import parallel_generate_sc_intervals
from run_spades import run_spades_parallel
from run_age import run_age_parallel
from generate_final_vcf import convert_metasv_bed_to_vcf
from fasta_utils import get_contigs
from genotype import parallel_genotype_intervals

FORMAT = '%(levelname)s %(asctime)-15s %(name)-20s %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT)
logger = logging.getLogger(__name__)

def create_dirs(dirlist):
    for dirname in dirlist:
        if not os.path.isdir(dirname):
            logger.info("Creating directory %s" % (dirname))
            os.makedirs(dirname)

def run_metasv(args):
    # Check if there is work to do
    if not (
                                args.pindel_vcf + args.breakdancer_vcf + args.breakseq_vcf + args.cnvnator_vcf + args.pindel_native + args.breakdancer_native + args.breakseq_native + args.cnvnator_native):
        logger.warning("Nothing to merge since no SV file specified")

    # Simple check for arguments
    if not args.disable_assembly:
        if not args.spades:
            logger.error("Spades executable not specified")
            return os.EX_USAGE

        if not args.age:
            logger.error("AGE executable not specified")
            return os.EX_USAGE

    # Create the directories for working
    bedtools_tmpdir = os.path.join(args.workdir, "bedtools")
    create_dirs([args.workdir, args.outdir, bedtools_tmpdir])

    # Reference handling
    if not os.path.isfile(args.reference + ".fai"):
        logger.error("Reference file %s is not indexed" % (args.reference))
        return 1

    fasta_handle = pysam.Fastafile(args.reference) if os.path.isfile(args.reference) else None
    contigs = get_contigs(args.reference)
    include_intervals = sorted(
        [SVInterval(contig.name, 0, contig.length, contig.name, "include", length=contig.length) for contig in contigs])

    # Generate the list of contigs to process
    contig_whitelist = set(args.chromosomes) if args.chromosomes else set([contig.name for contig in contigs])
    if args.keep_standard_contigs:
        contig_whitelist &= set(
            [str(i) for i in xrange(1, 23)] + ["chr%d" % (i) for i in xrange(1, 23)] + ["X", "Y", "MT", "chrX", "chrY",
                                                                                "chrM"])
    logger.info("Only SVs on the following contigs will be reported: %s" % (sorted(list(contig_whitelist))))

    # Load the intervals from different files
    vcf_name_list = [("CNVnator", args.cnvnator_vcf), ("Pindel", args.pindel_vcf),
                     ("BreakDancer", args.breakdancer_vcf),
                     ("BreakSeq", args.breakseq_vcf), ("HaplotypeCaller", args.gatk_vcf)]
    native_name_list = [("CNVnator", args.cnvnator_native, CNVnatorReader),
                        ("Pindel", args.pindel_native, PindelReader),
                        ("BreakSeq", args.breakseq_native, BreakSeqReader),
                        ("BreakDancer", args.breakdancer_native, BreakDancerReader)]

    tools = []
    intervals = {}
    sv_types = set()

    gap_intervals = []
    if args.filter_gaps:
        gaps = args.gaps if args.gaps else get_gaps_file(contig_whitelist)
        gap_intervals = sorted(load_gap_intervals(gaps))

    # Handles native input
    logger.info("Load native files")
    for toolname, nativename, svReader in native_name_list:
        # If no native file is given, ignore the tool
        if not nativename: continue

        tools.append(toolname)
        intervals[toolname] = defaultdict(list)

        for native_file in nativename:
            for record in svReader(native_file, svs_to_report=args.svs_to_report):
                interval = record.to_sv_interval()

                if not interval:
                    # This is the case for SVs we want to skip
                    continue
                if not interval_overlaps_interval_list(interval, gap_intervals) and interval.chrom in contig_whitelist:

                    # Check length
                    if interval.length < args.minsvlen:
                        continue

                    # Set wiggle
                    interval.wiggle = max(args.inswiggle if interval.sv_type == "INS" else 0, args.wiggle)

                    intervals[toolname][interval.sv_type].append(interval)

        sv_types |= set(intervals[toolname].keys())

    # Handles the VCF input cases, we will just deal with these cases
    logger.info("Load VCF files")
    for toolname, vcfname in vcf_name_list:
        # If no VCF is given, ignore the tool
        if not vcfname:
            continue

        tools.append(toolname)
        intervals[toolname] = {}

        vcf_list = []
        for vcffile in vcfname:
            if os.path.isdir(vcffile):
                logger.info("Will load from per-chromosome VCFs from directory %s for tool %s" % (vcffile, toolname))
                vcf_list += [os.path.join(vcffile, "%s.vcf.gz" % contig.name) for contig in contigs if
                             (not contig_whitelist or contig.name in contig_whitelist)]
            else:
                vcf_list.append(vcffile)

        for vcffile in vcf_list:
            load_intervals(vcffile, intervals[toolname], gap_intervals, include_intervals, toolname, contig_whitelist,
                           minsvlen=args.minsvlen, wiggle=args.wiggle, inswiggle=args.inswiggle,
                           svs_to_report=args.svs_to_report)
        sv_types |= set(intervals[toolname].keys())

    logger.info("SV types are %s" % (str(sv_types)))
    tool_merged_intervals = {}
    final_intervals = []

    # This will just output per-tool VCFs, no intra-tool merging is done yet
    if args.enable_per_tool_output:
        logger.info("Output per-tool VCFs")
        for toolname in intervals:
            tool_out = os.path.join(args.outdir, "%s.vcf" % (toolname.lower()))

            logger.info("Outputting single tool VCF for %s" % (str(toolname)))
            vcf_template_reader = vcf.Reader(open(os.path.join(mydir, "resources/template.vcf"), "r"))
            vcf_template_reader.samples = [args.sample]

            intervals_tool = []
            tool_out_fd = open(tool_out, "w")
            vcf_writer = vcf.Writer(tool_out_fd, vcf_template_reader)
            chr_intervals_tool = {contig.name: [] for contig in contigs}
            for sv_type in sv_types:
                if sv_type in intervals[toolname]:
                    intervals_tool.extend([copy.deepcopy(interval) for interval in intervals[toolname][sv_type]])
            for interval in intervals_tool:
                # Marghoob says that this is just to fill-in some metadata
                interval.do_validation(args.overlap_ratio)

                interval.fix_pos()
                chr_intervals_tool[interval.chrom].append(interval)

            for contig in contigs:
                chr_intervals_tool[contig.name].sort()
                for interval in chr_intervals_tool[contig.name]:
                    vcf_record = interval.to_vcf_record(fasta_handle, args.sample)
                    if vcf_record is not None:
                        vcf_writer.write_record(vcf_record)
            tool_out_fd.close()
            vcf_writer.close()
            logger.info("Indexing single tool VCF for %s" % (str(toolname)))
            pysam.tabix_index(tool_out, force=True, preset="vcf")

    # Do merging here
    logger.info("Do merging")
    for sv_type in sv_types:
        logger.info("Processing SVs of type %s" % sv_type)
        tool_merged_intervals[sv_type] = []

        # Do the intra-tool merging
        logger.info("Intra-tool Merging SVs of type %s" % sv_type)
        for tool in tools:
            logger.debug("Is %s in tool keys? %s" % (sv_type, str(intervals[tool].keys())))
            if sv_type not in intervals[tool]:
                logger.debug("%s not in tool %s" % (sv_type, tool))
                continue
            logger.info("First level merging for %s for tool %s" % (sv_type, tool))
            tool_merged_intervals[sv_type] += merge_intervals(intervals[tool][sv_type])

        # Do the inter-tool merging
        logger.info("Inter-tool Merging SVs of type %s" % sv_type)
        merged_intervals = merge_intervals(tool_merged_intervals[sv_type])

        # Intervals which overlap well with merged_intervals
        intervals1 = []
        # Intervals which do not overlap well with merged_intervals.
        # Used to filter out small intervals which got merged with large intervals
        intervals2 = []

        logger.info("Checking overlaps SVs of type %s" % sv_type)
        for interval in tool_merged_intervals[sv_type]:
            if interval_overlaps_interval_list(interval, merged_intervals, args.overlap_ratio, args.overlap_ratio):
                intervals2.append(interval)
            else:
                intervals1.append(interval)
        final_intervals.extend(merge_intervals(intervals1) + merge_intervals(intervals2))

    final_chr_intervals = {contig.name: [] for contig in contigs}
    for interval in final_intervals:
        if args.minsvlen <= interval.length <= args.maxsvlen:
            interval.do_validation(args.overlap_ratio)
            interval.fix_pos()
            final_chr_intervals[interval.chrom].append(interval)

    # This is the merged VCF without assembly, ok for deletions at this point
    logger.info("Output merged VCF without assembly ")
    vcf_template_reader = vcf.Reader(open(os.path.join(mydir, "resources/template.vcf"), "r"))
    vcf_template_reader.samples = [args.sample]
    preasm_vcf = os.path.join(args.workdir, "pre_asm.vcf")
    vcf_fd = open(preasm_vcf, "w")
    vcf_writer = vcf.Writer(vcf_fd, vcf_template_reader)

    final_stats = {}

    bed_intervals = []
    for contig in contigs:
        final_chr_intervals[contig.name].sort()
        for interval in final_chr_intervals[contig.name]:
            vcf_record = interval.to_vcf_record(fasta_handle)
            if vcf_record is not None:
                key = (interval.sv_type, "PASS" if interval.is_validated else "LowQual",
                       "PRECISE" if interval.is_precise else "IMPRECISE", tuple(sorted(list(interval.sources))))
                if key not in final_stats:
                    final_stats[key] = 0
                final_stats[key] += 1
                vcf_writer.write_record(vcf_record)
            bed_interval = interval.to_bed_interval(args.sample)
            if bed_interval is not None:
                bed_intervals.append(bed_interval)
    vcf_fd.close()
    vcf_writer.close()

    # Also save a BED file representation of the merged variants without assembly
    merged_bed = None
    if bed_intervals:
        merged_bed = os.path.join(args.workdir, "metasv.bed")
        pybedtools.BedTool(bed_intervals).saveas(merged_bed)

    for key in sorted(final_stats.keys()):
        logger.info(str(key) + ":" + str(final_stats[key]))

    final_vcf = os.path.join(args.outdir, "variants.vcf")

    # Run assembly here
    if not args.disable_assembly:
        logger.info("Running assembly")

        spades_tmpdir = os.path.join(args.workdir, "spades")
        age_tmpdir = os.path.join(args.workdir, "age")

        create_dirs([spades_tmpdir, age_tmpdir])

        assembly_bed = merged_bed

        # this does the improved assembly location finder with softclipped reads
        if args.boost_ins and "INS" in args.svs_to_assemble:
            logger.info("Generating intervals for insertions")
            assembly_bed = parallel_generate_sc_intervals([args.bam.name], list(contig_whitelist), merged_bed,
                                                          args.workdir,
                                                          num_threads=args.num_threads,
                                                          min_support=args.min_ins_support,
                                                          min_support_frac=args.min_ins_support_frac,
                                                          max_intervals=args.max_ins_intervals, min_mapq=args.min_mapq,
                                                          min_avg_base_qual=args.min_avg_base_qual,
                                                          min_soft_clip=args.min_soft_clip,
                                                          max_soft_clip=args.max_soft_clip,
                                                          max_nm=args.max_nm, min_matches=args.min_matches)
            logger.info("Generated intervals for assembly in %s" % assembly_bed)

        logger.info("Will run assembly now")

        assembled_fasta, ignored_bed = run_spades_parallel(bam=args.bam.name, spades=args.spades, bed=assembly_bed,
                                                           work=spades_tmpdir, pad=SPADES_PAD,
                                                           nthreads=args.num_threads,
                                                           chrs=list(contig_whitelist),
                                                           max_interval_size=args.spades_max_interval_size,
                                                           svs_to_assemble=args.svs_to_assemble,
                                                           stop_on_fail=args.stop_spades_on_fail,
                                                           max_read_pairs=args.extraction_max_read_pairs)
        breakpoints_bed = run_age_parallel(intervals_bed=assembly_bed, reference=args.reference,
                                           assembly=assembled_fasta,
                                           pad=AGE_PAD, age=args.age, chrs=list(contig_whitelist),
                                           nthreads=args.num_threads,
                                           min_contig_len=AGE_MIN_CONTIG_LENGTH, age_workdir=age_tmpdir)

        final_bed = os.path.join(args.workdir, "final.bed")
        if breakpoints_bed:
            if ignored_bed:
                pybedtools.BedTool(breakpoints_bed) \
                    .cat(pybedtools.BedTool(ignored_bed), postmerge=False) \
                    .sort().saveas(final_bed)
            else:
                pybedtools.BedTool(breakpoints_bed).saveas(final_bed)
        elif ignored_bed:
            pybedtools.BedTool(ignored_bed).sort().saveas(final_bed)
        else:
            final_bed = None

        genotyped_bed = parallel_genotype_intervals(final_bed, args.bam.name,
                                                    workdir=os.path.join(args.workdir, "genotyping"),
                                                    nthreads=args.num_threads, chromosomes=list(contig_whitelist),
                                                    window=args.gt_window, isize_mean=args.isize_mean,
                                                    isize_sd=args.isize_sd,
                                                    normal_frac_threshold=args.gt_normal_frac)

        logger.info("Output final VCF file")

        convert_metasv_bed_to_vcf(bedfile=genotyped_bed, vcf_out=final_vcf, sample=args.sample, pass_calls=False)
    else:
        shutil.copy(preasm_vcf, final_vcf)
        pysam.tabix_index(final_vcf, force=True, preset="vcf")

    logger.info("Clean up pybedtools")

    pybedtools.cleanup(remove_all=True)

    logger.info("All Done!")

    return os.EX_OK
<commit_msg>only filter based on svlen once length is calculated<commit_after>from collections import defaultdict
import shutil

from defaults import *
from vcf_utils import *
from sv_interval import SVInterval, get_gaps_file, interval_overlaps_interval_list, merge_intervals
from pindel_reader import PindelReader
from breakdancer_reader import BreakDancerReader
from breakseq_reader import BreakSeqReader
from cnvnator_reader import CNVnatorReader
from generate_sv_intervals import parallel_generate_sc_intervals
from run_spades import run_spades_parallel
from run_age import run_age_parallel
from generate_final_vcf import convert_metasv_bed_to_vcf
from fasta_utils import get_contigs
from genotype import parallel_genotype_intervals

FORMAT = '%(levelname)s %(asctime)-15s %(name)-20s %(message)s'
logging.basicConfig(level=logging.INFO, format=FORMAT)
logger = logging.getLogger(__name__)

def create_dirs(dirlist):
    for dirname in dirlist:
        if not os.path.isdir(dirname):
            logger.info("Creating directory %s" % (dirname))
            os.makedirs(dirname)

def run_metasv(args):
    # Check if there is work to do
    if not (
                                args.pindel_vcf + args.breakdancer_vcf + args.breakseq_vcf + args.cnvnator_vcf + args.pindel_native + args.breakdancer_native + args.breakseq_native + args.cnvnator_native):
        logger.warning("Nothing to merge since no SV file specified")

    # Simple check for arguments
    if not args.disable_assembly:
        if not args.spades:
            logger.error("Spades executable not specified")
            return os.EX_USAGE

        if not args.age:
            logger.error("AGE executable not specified")
            return os.EX_USAGE

    # Create the directories for working
    bedtools_tmpdir = os.path.join(args.workdir, "bedtools")
    create_dirs([args.workdir, args.outdir, bedtools_tmpdir])

    # Reference handling
    if not os.path.isfile(args.reference + ".fai"):
        logger.error("Reference file %s is not indexed" % (args.reference))
        return 1

    fasta_handle = pysam.Fastafile(args.reference) if os.path.isfile(args.reference) else None
    contigs = get_contigs(args.reference)
    include_intervals = sorted(
        [SVInterval(contig.name, 0, contig.length, contig.name, "include", length=contig.length) for contig in contigs])

    # Generate the list of contigs to process
    contig_whitelist = set(args.chromosomes) if args.chromosomes else set([contig.name for contig in contigs])
    if args.keep_standard_contigs:
        contig_whitelist &= set(
            [str(i) for i in xrange(1, 23)] + ["chr%d" % (i) for i in xrange(1, 23)] + ["X", "Y", "MT", "chrX", "chrY",
                                                                                "chrM"])
    logger.info("Only SVs on the following contigs will be reported: %s" % (sorted(list(contig_whitelist))))

    # Load the intervals from different files
    vcf_name_list = [("CNVnator", args.cnvnator_vcf), ("Pindel", args.pindel_vcf),
                     ("BreakDancer", args.breakdancer_vcf),
                     ("BreakSeq", args.breakseq_vcf), ("HaplotypeCaller", args.gatk_vcf)]
    native_name_list = [("CNVnator", args.cnvnator_native, CNVnatorReader),
                        ("Pindel", args.pindel_native, PindelReader),
                        ("BreakSeq", args.breakseq_native, BreakSeqReader),
                        ("BreakDancer", args.breakdancer_native, BreakDancerReader)]

    tools = []
    intervals = {}
    sv_types = set()

    gap_intervals = []
    if args.filter_gaps:
        gaps = args.gaps if args.gaps else get_gaps_file(contig_whitelist)
        gap_intervals = sorted(load_gap_intervals(gaps))

    # Handles native input
    logger.info("Load native files")
    for toolname, nativename, svReader in native_name_list:
        # If no native file is given, ignore the tool
        if not nativename: continue

        tools.append(toolname)
        intervals[toolname] = defaultdict(list)

        for native_file in nativename:
            for record in svReader(native_file, svs_to_report=args.svs_to_report):
                interval = record.to_sv_interval()

                if not interval:
                    # This is the case for SVs we want to skip
                    continue
                if not interval_overlaps_interval_list(interval, gap_intervals) and interval.chrom in contig_whitelist:

                    # Check length
                    if interval.length < args.minsvlen:
                        continue

                    # Set wiggle
                    interval.wiggle = max(args.inswiggle if interval.sv_type == "INS" else 0, args.wiggle)

                    intervals[toolname][interval.sv_type].append(interval)

        sv_types |= set(intervals[toolname].keys())

    # Handles the VCF input cases, we will just deal with these cases
    logger.info("Load VCF files")
    for toolname, vcfname in vcf_name_list:
        # If no VCF is given, ignore the tool
        if not vcfname:
            continue

        tools.append(toolname)
        intervals[toolname] = {}

        vcf_list = []
        for vcffile in vcfname:
            if os.path.isdir(vcffile):
                logger.info("Will load from per-chromosome VCFs from directory %s for tool %s" % (vcffile, toolname))
                vcf_list += [os.path.join(vcffile, "%s.vcf.gz" % contig.name) for contig in contigs if
                             (not contig_whitelist or contig.name in contig_whitelist)]
            else:
                vcf_list.append(vcffile)

        for vcffile in vcf_list:
            load_intervals(vcffile, intervals[toolname], gap_intervals, include_intervals, toolname, contig_whitelist,
                           minsvlen=args.minsvlen, wiggle=args.wiggle, inswiggle=args.inswiggle,
                           svs_to_report=args.svs_to_report)
        sv_types |= set(intervals[toolname].keys())

    logger.info("SV types are %s" % (str(sv_types)))
    tool_merged_intervals = {}
    final_intervals = []

    # This will just output per-tool VCFs, no intra-tool merging is done yet
    if args.enable_per_tool_output:
        logger.info("Output per-tool VCFs")
        for toolname in intervals:
            tool_out = os.path.join(args.outdir, "%s.vcf" % (toolname.lower()))

            logger.info("Outputting single tool VCF for %s" % (str(toolname)))
            vcf_template_reader = vcf.Reader(open(os.path.join(mydir, "resources/template.vcf"), "r"))
            vcf_template_reader.samples = [args.sample]

            intervals_tool = []
            tool_out_fd = open(tool_out, "w")
            vcf_writer = vcf.Writer(tool_out_fd, vcf_template_reader)
            chr_intervals_tool = {contig.name: [] for contig in contigs}
            for sv_type in sv_types:
                if sv_type in intervals[toolname]:
                    intervals_tool.extend([copy.deepcopy(interval) for interval in intervals[toolname][sv_type]])
            for interval in intervals_tool:
                # Marghoob says that this is just to fill-in some metadata
                interval.do_validation(args.overlap_ratio)

                interval.fix_pos()
                chr_intervals_tool[interval.chrom].append(interval)

            for contig in contigs:
                chr_intervals_tool[contig.name].sort()
                for interval in chr_intervals_tool[contig.name]:
                    vcf_record = interval.to_vcf_record(fasta_handle, args.sample)
                    if vcf_record is not None:
                        vcf_writer.write_record(vcf_record)
            tool_out_fd.close()
            vcf_writer.close()
            logger.info("Indexing single tool VCF for %s" % (str(toolname)))
            pysam.tabix_index(tool_out, force=True, preset="vcf")

    # Do merging here
    logger.info("Do merging")
    for sv_type in sv_types:
        logger.info("Processing SVs of type %s" % sv_type)
        tool_merged_intervals[sv_type] = []

        # Do the intra-tool merging
        logger.info("Intra-tool Merging SVs of type %s" % sv_type)
        for tool in tools:
            logger.debug("Is %s in tool keys? %s" % (sv_type, str(intervals[tool].keys())))
            if sv_type not in intervals[tool]:
                logger.debug("%s not in tool %s" % (sv_type, tool))
                continue
            logger.info("First level merging for %s for tool %s" % (sv_type, tool))
            tool_merged_intervals[sv_type] += merge_intervals(intervals[tool][sv_type])

        # Do the inter-tool merging
        logger.info("Inter-tool Merging SVs of type %s" % sv_type)
        merged_intervals = merge_intervals(tool_merged_intervals[sv_type])

        # Intervals which overlap well with merged_intervals
        intervals1 = []
        # Intervals which do not overlap well with merged_intervals.
        # Used to filter out small intervals which got merged with large intervals
        intervals2 = []

        logger.info("Checking overlaps SVs of type %s" % sv_type)
        for interval in tool_merged_intervals[sv_type]:
            if interval_overlaps_interval_list(interval, merged_intervals, args.overlap_ratio, args.overlap_ratio):
                intervals2.append(interval)
            else:
                intervals1.append(interval)
        final_intervals.extend(merge_intervals(intervals1) + merge_intervals(intervals2))

    final_chr_intervals = {contig.name: [] for contig in contigs}
    for interval in final_intervals:
        interval.do_validation(args.overlap_ratio)
        interval.fix_pos()
        if args.minsvlen <= interval.length <= args.maxsvlen:
            final_chr_intervals[interval.chrom].append(interval)

    # This is the merged VCF without assembly, ok for deletions at this point
    logger.info("Output merged VCF without assembly ")
    vcf_template_reader = vcf.Reader(open(os.path.join(mydir, "resources/template.vcf"), "r"))
    vcf_template_reader.samples = [args.sample]
    preasm_vcf = os.path.join(args.workdir, "pre_asm.vcf")
    vcf_fd = open(preasm_vcf, "w")
    vcf_writer = vcf.Writer(vcf_fd, vcf_template_reader)

    final_stats = {}

    bed_intervals = []
    for contig in contigs:
        final_chr_intervals[contig.name].sort()
        for interval in final_chr_intervals[contig.name]:
            vcf_record = interval.to_vcf_record(fasta_handle)
            if vcf_record is not None:
                key = (interval.sv_type, "PASS" if interval.is_validated else "LowQual",
                       "PRECISE" if interval.is_precise else "IMPRECISE", tuple(sorted(list(interval.sources))))
                if key not in final_stats:
                    final_stats[key] = 0
                final_stats[key] += 1
                vcf_writer.write_record(vcf_record)
            bed_interval = interval.to_bed_interval(args.sample)
            if bed_interval is not None:
                bed_intervals.append(bed_interval)
    vcf_fd.close()
    vcf_writer.close()

    # Also save a BED file representation of the merged variants without assembly
    merged_bed = None
    if bed_intervals:
        merged_bed = os.path.join(args.workdir, "metasv.bed")
        pybedtools.BedTool(bed_intervals).saveas(merged_bed)

    for key in sorted(final_stats.keys()):
        logger.info(str(key) + ":" + str(final_stats[key]))

    final_vcf = os.path.join(args.outdir, "variants.vcf")

    # Run assembly here
    if not args.disable_assembly:
        logger.info("Running assembly")

        spades_tmpdir = os.path.join(args.workdir, "spades")
        age_tmpdir = os.path.join(args.workdir, "age")

        create_dirs([spades_tmpdir, age_tmpdir])

        assembly_bed = merged_bed

        # this does the improved assembly location finder with softclipped reads
        if args.boost_ins and "INS" in args.svs_to_assemble:
            logger.info("Generating intervals for insertions")
            assembly_bed = parallel_generate_sc_intervals([args.bam.name], list(contig_whitelist), merged_bed,
                                                          args.workdir,
                                                          num_threads=args.num_threads,
                                                          min_support=args.min_ins_support,
                                                          min_support_frac=args.min_ins_support_frac,
                                                          max_intervals=args.max_ins_intervals, min_mapq=args.min_mapq,
                                                          min_avg_base_qual=args.min_avg_base_qual,
                                                          min_soft_clip=args.min_soft_clip,
                                                          max_soft_clip=args.max_soft_clip,
                                                          max_nm=args.max_nm, min_matches=args.min_matches)
            logger.info("Generated intervals for assembly in %s" % assembly_bed)

        logger.info("Will run assembly now")

        assembled_fasta, ignored_bed = run_spades_parallel(bam=args.bam.name, spades=args.spades, bed=assembly_bed,
                                                           work=spades_tmpdir, pad=SPADES_PAD,
                                                           nthreads=args.num_threads,
                                                           chrs=list(contig_whitelist),
                                                           max_interval_size=args.spades_max_interval_size,
                                                           svs_to_assemble=args.svs_to_assemble,
                                                           stop_on_fail=args.stop_spades_on_fail,
                                                           max_read_pairs=args.extraction_max_read_pairs)
        breakpoints_bed = run_age_parallel(intervals_bed=assembly_bed, reference=args.reference,
                                           assembly=assembled_fasta,
                                           pad=AGE_PAD, age=args.age, chrs=list(contig_whitelist),
                                           nthreads=args.num_threads,
                                           min_contig_len=AGE_MIN_CONTIG_LENGTH, age_workdir=age_tmpdir)

        final_bed = os.path.join(args.workdir, "final.bed")
        if breakpoints_bed:
            if ignored_bed:
                pybedtools.BedTool(breakpoints_bed) \
                    .cat(pybedtools.BedTool(ignored_bed), postmerge=False) \
                    .sort().saveas(final_bed)
            else:
                pybedtools.BedTool(breakpoints_bed).saveas(final_bed)
        elif ignored_bed:
            pybedtools.BedTool(ignored_bed).sort().saveas(final_bed)
        else:
            final_bed = None

        genotyped_bed = parallel_genotype_intervals(final_bed, args.bam.name,
                                                    workdir=os.path.join(args.workdir, "genotyping"),
                                                    nthreads=args.num_threads, chromosomes=list(contig_whitelist),
                                                    window=args.gt_window, isize_mean=args.isize_mean,
                                                    isize_sd=args.isize_sd,
                                                    normal_frac_threshold=args.gt_normal_frac)

        logger.info("Output final VCF file")

        convert_metasv_bed_to_vcf(bedfile=genotyped_bed, vcf_out=final_vcf, sample=args.sample, pass_calls=False)
    else:
        shutil.copy(preasm_vcf, final_vcf)
        pysam.tabix_index(final_vcf, force=True, preset="vcf")

    logger.info("Clean up pybedtools")

    pybedtools.cleanup(remove_all=True)

    logger.info("All Done!")

    return os.EX_OK
<|endoftext|>


<commit_before>consoles = {
    'nes': {
        'name': 'Nintendo Entertainment System',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("a", "A"),
            ("b", "B"),
            ("start", "Start"),
            ("select", "Select"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v', '-L',
            '/usr/lib/libretro/quicknes_libretro.so'
        ],
    },
    'pce': {
        'name': 'PC Engine & SuperGrafx',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v', '-L',
            '/usr/lib/libretro/mednafen_supergrafx_libretro.so'
        ],
    },
    'snes': {
        'name': 'Super Nintendo',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v', '-L',
            '/usr/lib/libretro/snes9x_next_libretro.so'
        ],
    },
    'md': {
        'name': 'Mega Drive / Genesis',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v', '-L',
            '/usr/lib/libretro/genesis_plus_gx_libretro.so'
        ],
    },
}
<commit_msg>Force retroarch to use the RO system config<commit_after>consoles = {
    'nes': {
        'name': 'Nintendo Entertainment System',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("a", "A"),
            ("b", "B"),
            ("start", "Start"),
            ("select", "Select"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v',
            '-c', '/etc/retroarch.cfg',
            '-L', '/usr/lib/libretro/quicknes_libretro.so'
        ],
    },
    'pce': {
        'name': 'PC Engine & SuperGrafx',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v',
            '-c', '/etc/retroarch.cfg',
            '-L', '/usr/lib/libretro/mednafen_supergrafx_libretro.so'
        ],
    },
    'snes': {
        'name': 'Super Nintendo',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v',
            '-c', '/etc/retroarch.cfg',
            '-L', '/usr/lib/libretro/snes9x_next_libretro.so'
        ],
    },
    'md': {
        'name': 'Mega Drive / Genesis',
        'controls': [
            ("up", "Up"),
            ("down", "Down"),
            ("left", "Left"),
            ("right", "Right"),
            ("y", "A"),
            ("b", "B"),
            ("a", "C"),
            ("l", "X"),
            ("x", "Y"),
            ("r", "Z"),
            ("start", "Start"),
        ],
        'exec': [
            'sudo', '-u', 'meldnafen', 'retroarch', '-v',
            '-c', '/etc/retroarch.cfg',
            '-L', '/usr/lib/libretro/genesis_plus_gx_libretro.so'
        ],
    },
}
<|endoftext|>