<commit_before># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------

##############################################################################
# Load font into texture

from __future__ import division

import numpy as np
from copy import deepcopy

from ._sdf import SDFRenderer
from ....gloo import TextureAtlas, set_state, IndexBuffer, VertexBuffer
from ....gloo.wrappers import _check_valid
from ....ext.six import string_types
from ....util.fonts import _load_glyph
from ...shaders import ModularProgram
from ....color import Color
from ..visual import Visual

class TextureFont(object):
    """Gather a set of glyphs relative to a given font name and size

    Parameters
    ----------
    font : dict
        Dict with entries "face", "size", "bold", "italic".
    renderer : instance of SDFRenderer
        SDF renderer to use.
    """
    def __init__(self, font, renderer):
        self._atlas = TextureAtlas()
        self._atlas.wrapping = 'clamp_to_edge'
        self._renderer = renderer
        self._font = deepcopy(font)
        self._font['size'] = 256  # use high resolution point size for SDF
        self._lowres_size = 64  # end at this point size for storage
        assert (self._font['size'] % self._lowres_size) == 0
        # spread/border at the high-res for SDF calculation; must be chosen
        # relative to fragment_insert.glsl multiplication factor to ensure we
        # get to zero at the edges of characters
        self._spread = 32
        assert self._spread % self.ratio == 0
        self._glyphs = {}

    @property
    def ratio(self):
        """Ratio of the initial high-res to final stored low-res glyph"""
        return self._font['size'] // self._lowres_size

    @property
    def slop(self):
        """Extra space along each glyph edge due to SDF borders"""
        return self._spread // self.ratio

    def __getitem__(self, char):
        if not (isinstance(char, string_types) and len(char) == 1):
            raise TypeError('index must be a 1-character string')
        if char not in self._glyphs:
            self._load_char(char)
        return self._glyphs[char]

    def _load_char(self, char):
        """Build and store a glyph corresponding to an individual character

        Parameters:
        -----------
        char : str
            A single character to be represented.
        """
        assert isinstance(char, string_types) and len(char) == 1
        assert char not in self._glyphs
        # load new glyph data from font
        _load_glyph(self._font, char, self._glyphs)
        # put new glyph into the texture
        glyph = self._glyphs[char]
        bitmap = glyph['bitmap']

        # convert to padded array
        data = np.zeros((bitmap.shape[0] + 2*self._spread,
                         bitmap.shape[1] + 2*self._spread), np.uint8)
        data[self._spread:-self._spread, self._spread:-self._spread] = bitmap

        # Store, while scaling down to proper size
        height = data.shape[0] // self.ratio
        width = data.shape[1] // self.ratio
        region = self._atlas.get_free_region(width + 2, height + 2)
        if region is None:
            raise RuntimeError('Cannot store glyph')
        x, y, w, h = region
        x, y, w, h = x + 1, y + 1, w - 2, h - 2

        self._renderer.render_to_texture(data, self._atlas, (x, y), (w, h))
        u0 = x / float(self._atlas.shape[1])
        v0 = y / float(self._atlas.shape[0])
        u1 = (x+w) / float(self._atlas.shape[1])
        v1 = (y+h) / float(self._atlas.shape[0])
        texcoords = (u0, v0, u1, v1)
        glyph.update(dict(size=(w, h), texcoords=texcoords))

class FontManager(object):
    def __init__(self):
        self._fonts = {}
        self._renderer = SDFRenderer()

    def get_font(self, face, bold=False, italic=False):
        """Get a font described by face and size"""
        key = '%s-%s-%s' % (face, bold, italic)
        if key not in self._fonts:
            font = dict(face=face, bold=bold, italic=italic)
            self._fonts[key] = TextureFont(font, self._renderer)
        return self._fonts[key]

##############################################################################
# The visual

def _text_to_vbo(text, font, anchor_x, anchor_y, lowres_size):
    """Convert text characters to VBO"""
    text_vtype = np.dtype([('a_position', 'f4', 2),
                           ('a_texcoord', 'f4', 2)])
    vertices = np.zeros(len(text) * 4, dtype=text_vtype)
    prev = None
    width = height = ascender = descender = 0
    ratio, slop = 1. / font.ratio, font.slop
    x_off = -slop
    for ii, char in enumerate(text):
        glyph = font[char]
        kerning = glyph['kerning'].get(prev, 0.) * ratio
        x0 = x_off + glyph['offset'][0] * ratio + kerning
        y0 = glyph['offset'][1] * ratio + slop
        x1 = x0 + glyph['size'][0]
        y1 = y0 - glyph['size'][1]
        u0, v0, u1, v1 = glyph['texcoords']
        position = [[x0, y0], [x0, y1], [x1, y1], [x1, y0]]
        texcoords = [[u0, v0], [u0, v1], [u1, v1], [u1, v0]]
        vi = ii * 4
        vertices['a_position'][vi:vi+4] = position
        vertices['a_texcoord'][vi:vi+4] = texcoords
        x_move = glyph['advance'] * ratio + kerning
        x_off += x_move
        ascender = max(ascender, y0 - slop)
        descender = min(descender, y1 + slop)
        width += x_move
        height = max(height, glyph['size'][1] - 2*slop)
        prev = char

    # Tight bounding box (loose would be width, font.height /.asc / .desc)
    width -= glyph['advance'] * ratio - (glyph['size'][0] - 2*slop)
    dx = dy = 0
    if anchor_y == 'top':
        dy = -ascender
    elif anchor_y in ('center', 'middle'):
        dy = -(height / 2 + descender)
    elif anchor_y == 'bottom':
        dy = -descender
    # Already referenced to baseline
    # elif anchor_y == 'baseline':
    #     dy = -descender
    if anchor_x == 'right':
        dx = -width
    elif anchor_x == 'center':
        dx = -width / 2.
    vertices['a_position'] += (dx, dy)
    vertices['a_position'] /= lowres_size
    return VertexBuffer(vertices)

class Text(Visual):
    """Visual that displays text

    Parameters
    ----------
    text : str
        Text to display.
    color : instance of Color
        Color to use.
    bold : bool
        Bold face.
    italic : bool
        Italic face.
    face : str
        Font face to use.
    font_size : float
        Point size to use.
    pos : tuple
        Position (x, y) of the text.
    rotation : float
        Rotation (in degrees) of the text clockwise.
    anchor_x : str
        Horizontal text anchor.
    anchor_y : str
        Vertical text anchor.
    parent : instance of Entity
        The parent of the Text visual.
    """

    VERTEX_SHADER = """
        uniform vec2 u_pos;  // anchor position
        uniform vec2 u_scale;  // to scale to pixel units
        uniform float u_rotation;  // rotation in rad
        attribute vec2 a_position; // in point units
        attribute vec2 a_texcoord;

        varying vec2 v_texcoord;

        void main(void) {
            vec4 pos = $transform(vec4(u_pos, 0.0, 1.0));
            mat2 rot = mat2(cos(u_rotation), -sin(u_rotation),
                            sin(u_rotation), cos(u_rotation));
            gl_Position = pos + vec4(rot * a_position * u_scale, 0., 0.);
            v_texcoord = a_texcoord;
        }
        """

    FRAGMENT_SHADER = """
        uniform sampler2D u_font_atlas;
        uniform vec4 u_color;

        varying vec2 v_texcoord;
        const float center = 0.5;

        void main(void) {
            vec4 color = u_color;
            vec2 uv = v_texcoord.xy;
            vec4 rgb = texture2D(u_font_atlas, uv);
            float distance = rgb.r;
            float width = fwidth(distance);
            float alpha = smoothstep(center - width, center + width, distance);
            gl_FragColor = vec4(color.rgb, color.a * alpha);
        }
        """

    def __init__(self, text, color='black', bold=False,
                 italic=False, face='OpenSans', font_size=12, pos=(0, 0),
                 rotation=0., anchor_x='center', anchor_y='center', **kwargs):
        Visual.__init__(self, **kwargs)
        # Check input
        assert isinstance(text, string_types)
        valid_keys = ('top', 'center', 'middle', 'baseline', 'bottom')
        _check_valid('anchor_y', anchor_y, valid_keys)
        valid_keys = ('left', 'center', 'right')
        _check_valid('anchor_x', anchor_x, valid_keys)
        # Init font handling stuff
        self._font_manager = FontManager()
        self._font = self._font_manager.get_font(face, bold, italic)
        self._program = ModularProgram(self.VERTEX_SHADER,
                                       self.FRAGMENT_SHADER)
        self._vertices = None
        self._anchors = (anchor_x, anchor_y)
        # Init text properties
        self.color = color
        self.text = text
        self.font_size = font_size
        self.pos = pos
        self.rotation = float(rotation)

    @property
    def text(self):
        """The text string"""
        return self._text

    @text.setter
    def text(self, text):
        assert isinstance(text, string_types)
        self._text = text
        self._vertices = None

    @property
    def font_size(self):
        """ The font size (in points) of the text
        """
        return self._font_size

    @font_size.setter
    def font_size(self, size):
        self._font_size = max(0.0, float(size))

    @property
    def color(self):
        """ The color of the text
        """
        return self._color

    @color.setter
    def color(self, color):
        self._color = Color(color)

    @property
    def pos(self):
        """ The position of the text anchor in the local coordinate frame
        """
        return self._pos

    @pos.setter
    def pos(self, pos):
        pos = [float(p) for p in pos]
        assert len(pos) == 2
        self._pos = tuple(pos)

    def draw(self, event=None):
        # attributes / uniforms are not available until program is built
        if len(self.text) == 0:
            return
        if self._vertices is None:
            # we delay creating vertices because it requires a context,
            # which may or may not exist when the object is initialized
            self._vertices = _text_to_vbo(self._text, self._font,
                                          self._anchors[0], self._anchors[1],
                                          self._font._lowres_size)
            idx = (np.array([0, 1, 2, 0, 2, 3], np.uint32) +
                   np.arange(0, 4*len(self._text), 4,
                             dtype=np.uint32)[:, np.newaxis])
            self._ib = IndexBuffer(idx.ravel())

        if event is not None:
            xform = event.render_transform.shader_map()
            self._program.vert['transform'] = xform
            px_scale = event.canvas.framebuffer.transform.scale
        else:
            self._program.vert['transform'] = self.transform.shader_map()
            # Rather arbitrary scale. With size=12 it takes up ~1/10 of space
            px_scale = 0.01, 0.01

        self._program.prepare()  # Force ModularProgram to set shaders
        # todo: do some testing to verify that the scaling is correct
        ps = (self._font_size / 72) * 92
        self._program['u_scale'] = ps * px_scale[0], ps * px_scale[1]
        self._program['u_rotation'] = self.rotation * np.pi / 180.
        self._program['u_pos'] = self._pos
        self._program['u_color'] = self._color.rgba
        self._program['u_font_atlas'] = self._font._atlas
        self._program.bind(self._vertices)
        set_state(blend=True, depth_test=False,
                  blend_func=('src_alpha', 'one_minus_src_alpha'))
        self._program.draw('triangles', self._ib)
<commit_msg>FIX: Make property<commit_after># -*- coding: utf-8 -*-
# -----------------------------------------------------------------------------
# Copyright (c) 2014, Vispy Development Team. All Rights Reserved.
# Distributed under the (new) BSD License. See LICENSE.txt for more info.
# -----------------------------------------------------------------------------

##############################################################################
# Load font into texture

from __future__ import division

import numpy as np
from copy import deepcopy

from ._sdf import SDFRenderer
from ....gloo import TextureAtlas, set_state, IndexBuffer, VertexBuffer
from ....gloo.wrappers import _check_valid
from ....ext.six import string_types
from ....util.fonts import _load_glyph
from ...shaders import ModularProgram
from ....color import Color
from ..visual import Visual

class TextureFont(object):
    """Gather a set of glyphs relative to a given font name and size

    Parameters
    ----------
    font : dict
        Dict with entries "face", "size", "bold", "italic".
    renderer : instance of SDFRenderer
        SDF renderer to use.
    """
    def __init__(self, font, renderer):
        self._atlas = TextureAtlas()
        self._atlas.wrapping = 'clamp_to_edge'
        self._renderer = renderer
        self._font = deepcopy(font)
        self._font['size'] = 256  # use high resolution point size for SDF
        self._lowres_size = 64  # end at this point size for storage
        assert (self._font['size'] % self._lowres_size) == 0
        # spread/border at the high-res for SDF calculation; must be chosen
        # relative to fragment_insert.glsl multiplication factor to ensure we
        # get to zero at the edges of characters
        self._spread = 32
        assert self._spread % self.ratio == 0
        self._glyphs = {}

    @property
    def ratio(self):
        """Ratio of the initial high-res to final stored low-res glyph"""
        return self._font['size'] // self._lowres_size

    @property
    def slop(self):
        """Extra space along each glyph edge due to SDF borders"""
        return self._spread // self.ratio

    def __getitem__(self, char):
        if not (isinstance(char, string_types) and len(char) == 1):
            raise TypeError('index must be a 1-character string')
        if char not in self._glyphs:
            self._load_char(char)
        return self._glyphs[char]

    def _load_char(self, char):
        """Build and store a glyph corresponding to an individual character

        Parameters:
        -----------
        char : str
            A single character to be represented.
        """
        assert isinstance(char, string_types) and len(char) == 1
        assert char not in self._glyphs
        # load new glyph data from font
        _load_glyph(self._font, char, self._glyphs)
        # put new glyph into the texture
        glyph = self._glyphs[char]
        bitmap = glyph['bitmap']

        # convert to padded array
        data = np.zeros((bitmap.shape[0] + 2*self._spread,
                         bitmap.shape[1] + 2*self._spread), np.uint8)
        data[self._spread:-self._spread, self._spread:-self._spread] = bitmap

        # Store, while scaling down to proper size
        height = data.shape[0] // self.ratio
        width = data.shape[1] // self.ratio
        region = self._atlas.get_free_region(width + 2, height + 2)
        if region is None:
            raise RuntimeError('Cannot store glyph')
        x, y, w, h = region
        x, y, w, h = x + 1, y + 1, w - 2, h - 2

        self._renderer.render_to_texture(data, self._atlas, (x, y), (w, h))
        u0 = x / float(self._atlas.shape[1])
        v0 = y / float(self._atlas.shape[0])
        u1 = (x+w) / float(self._atlas.shape[1])
        v1 = (y+h) / float(self._atlas.shape[0])
        texcoords = (u0, v0, u1, v1)
        glyph.update(dict(size=(w, h), texcoords=texcoords))

class FontManager(object):
    def __init__(self):
        self._fonts = {}
        self._renderer = SDFRenderer()

    def get_font(self, face, bold=False, italic=False):
        """Get a font described by face and size"""
        key = '%s-%s-%s' % (face, bold, italic)
        if key not in self._fonts:
            font = dict(face=face, bold=bold, italic=italic)
            self._fonts[key] = TextureFont(font, self._renderer)
        return self._fonts[key]

##############################################################################
# The visual

def _text_to_vbo(text, font, anchor_x, anchor_y, lowres_size):
    """Convert text characters to VBO"""
    text_vtype = np.dtype([('a_position', 'f4', 2),
                           ('a_texcoord', 'f4', 2)])
    vertices = np.zeros(len(text) * 4, dtype=text_vtype)
    prev = None
    width = height = ascender = descender = 0
    ratio, slop = 1. / font.ratio, font.slop
    x_off = -slop
    for ii, char in enumerate(text):
        glyph = font[char]
        kerning = glyph['kerning'].get(prev, 0.) * ratio
        x0 = x_off + glyph['offset'][0] * ratio + kerning
        y0 = glyph['offset'][1] * ratio + slop
        x1 = x0 + glyph['size'][0]
        y1 = y0 - glyph['size'][1]
        u0, v0, u1, v1 = glyph['texcoords']
        position = [[x0, y0], [x0, y1], [x1, y1], [x1, y0]]
        texcoords = [[u0, v0], [u0, v1], [u1, v1], [u1, v0]]
        vi = ii * 4
        vertices['a_position'][vi:vi+4] = position
        vertices['a_texcoord'][vi:vi+4] = texcoords
        x_move = glyph['advance'] * ratio + kerning
        x_off += x_move
        ascender = max(ascender, y0 - slop)
        descender = min(descender, y1 + slop)
        width += x_move
        height = max(height, glyph['size'][1] - 2*slop)
        prev = char

    # Tight bounding box (loose would be width, font.height /.asc / .desc)
    width -= glyph['advance'] * ratio - (glyph['size'][0] - 2*slop)
    dx = dy = 0
    if anchor_y == 'top':
        dy = -ascender
    elif anchor_y in ('center', 'middle'):
        dy = -(height / 2 + descender)
    elif anchor_y == 'bottom':
        dy = -descender
    # Already referenced to baseline
    # elif anchor_y == 'baseline':
    #     dy = -descender
    if anchor_x == 'right':
        dx = -width
    elif anchor_x == 'center':
        dx = -width / 2.
    vertices['a_position'] += (dx, dy)
    vertices['a_position'] /= lowres_size
    return VertexBuffer(vertices)

class Text(Visual):
    """Visual that displays text

    Parameters
    ----------
    text : str
        Text to display.
    color : instance of Color
        Color to use.
    bold : bool
        Bold face.
    italic : bool
        Italic face.
    face : str
        Font face to use.
    font_size : float
        Point size to use.
    pos : tuple
        Position (x, y) of the text.
    rotation : float
        Rotation (in degrees) of the text clockwise.
    anchor_x : str
        Horizontal text anchor.
    anchor_y : str
        Vertical text anchor.
    parent : instance of Entity
        The parent of the Text visual.
    """

    VERTEX_SHADER = """
        uniform vec2 u_pos;  // anchor position
        uniform vec2 u_scale;  // to scale to pixel units
        uniform float u_rotation;  // rotation in rad
        attribute vec2 a_position; // in point units
        attribute vec2 a_texcoord;

        varying vec2 v_texcoord;

        void main(void) {
            vec4 pos = $transform(vec4(u_pos, 0.0, 1.0));
            mat2 rot = mat2(cos(u_rotation), -sin(u_rotation),
                            sin(u_rotation), cos(u_rotation));
            gl_Position = pos + vec4(rot * a_position * u_scale, 0., 0.);
            v_texcoord = a_texcoord;
        }
        """

    FRAGMENT_SHADER = """
        uniform sampler2D u_font_atlas;
        uniform vec4 u_color;

        varying vec2 v_texcoord;
        const float center = 0.5;

        void main(void) {
            vec4 color = u_color;
            vec2 uv = v_texcoord.xy;
            vec4 rgb = texture2D(u_font_atlas, uv);
            float distance = rgb.r;
            float width = fwidth(distance);
            float alpha = smoothstep(center - width, center + width, distance);
            gl_FragColor = vec4(color.rgb, color.a * alpha);
        }
        """

    def __init__(self, text, color='black', bold=False,
                 italic=False, face='OpenSans', font_size=12, pos=(0, 0),
                 rotation=0., anchor_x='center', anchor_y='center', **kwargs):
        Visual.__init__(self, **kwargs)
        # Check input
        assert isinstance(text, string_types)
        valid_keys = ('top', 'center', 'middle', 'baseline', 'bottom')
        _check_valid('anchor_y', anchor_y, valid_keys)
        valid_keys = ('left', 'center', 'right')
        _check_valid('anchor_x', anchor_x, valid_keys)
        # Init font handling stuff
        self._font_manager = FontManager()
        self._font = self._font_manager.get_font(face, bold, italic)
        self._program = ModularProgram(self.VERTEX_SHADER,
                                       self.FRAGMENT_SHADER)
        self._vertices = None
        self._anchors = (anchor_x, anchor_y)
        # Init text properties
        self.color = color
        self.text = text
        self.font_size = font_size
        self.pos = pos
        self.rotation = rotation

    @property
    def text(self):
        """The text string"""
        return self._text

    @text.setter
    def text(self, text):
        assert isinstance(text, string_types)
        self._text = text
        self._vertices = None

    @property
    def font_size(self):
        """ The font size (in points) of the text
        """
        return self._font_size

    @font_size.setter
    def font_size(self, size):
        self._font_size = max(0.0, float(size))

    @property
    def color(self):
        """ The color of the text
        """
        return self._color

    @color.setter
    def color(self, color):
        self._color = Color(color)

    @property
    def rotation(self):
        """ The rotation of the text (clockwise, in degrees)
        """
        return self._rotation * 180. / np.pi

    @rotation.setter
    def rotation(self, rotation):
        self._rotation = float(rotation) * np.pi / 180.

    @property
    def pos(self):
        """ The position of the text anchor in the local coordinate frame
        """
        return self._pos

    @pos.setter
    def pos(self, pos):
        pos = [float(p) for p in pos]
        assert len(pos) == 2
        self._pos = tuple(pos)

    def draw(self, event=None):
        # attributes / uniforms are not available until program is built
        if len(self.text) == 0:
            return
        if self._vertices is None:
            # we delay creating vertices because it requires a context,
            # which may or may not exist when the object is initialized
            self._vertices = _text_to_vbo(self._text, self._font,
                                          self._anchors[0], self._anchors[1],
                                          self._font._lowres_size)
            idx = (np.array([0, 1, 2, 0, 2, 3], np.uint32) +
                   np.arange(0, 4*len(self._text), 4,
                             dtype=np.uint32)[:, np.newaxis])
            self._ib = IndexBuffer(idx.ravel())

        if event is not None:
            xform = event.render_transform.shader_map()
            self._program.vert['transform'] = xform
            px_scale = event.canvas.framebuffer.transform.scale
        else:
            self._program.vert['transform'] = self.transform.shader_map()
            # Rather arbitrary scale. With size=12 it takes up ~1/10 of space
            px_scale = 0.01, 0.01

        self._program.prepare()  # Force ModularProgram to set shaders
        # todo: do some testing to verify that the scaling is correct
        ps = (self._font_size / 72) * 92
        self._program['u_scale'] = ps * px_scale[0], ps * px_scale[1]
        self._program['u_rotation'] = self._rotation
        self._program['u_pos'] = self._pos
        self._program['u_color'] = self._color.rgba
        self._program['u_font_atlas'] = self._font._atlas
        self._program.bind(self._vertices)
        set_state(blend=True, depth_test=False,
                  blend_func=('src_alpha', 'one_minus_src_alpha'))
        self._program.draw('triangles', self._ib)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  trapeza-import.py
#
#  Copyright 2013-2014 David Reed <david@ktema.org>
#  This file is available under the terms of the MIT License.
#

from itertools import groupby
from operator import itemgetter, attrgetter

import hashlib
import io
import os
import sys
import pickle
import tempfile
import trapeza
import trapeza.formats
import trapeza.match
from flask import Flask, request, session, make_response, render_template, flash, redirect, url_for

# Accept three Source objects. Run match_incoming_records() on them.
# Render a template showing the user matches and allowing them to select a
# matching record.
# On selection, run diff on the selected records (comparing the columns that are
# mapped by the profile) and offer the user the chance to select one of the two versions
# of each field or to enter their own data.
# On output, generate a copy of the input sheet with a unique ID column added specifying
# the matched record in the database. For each diffed column, if the user did not choose
# to retain existing data, include the user-input value or the value from incoming.
# Maintain all other columns in the original input sheet unaltered for import.

app = Flask(__name__)
app.secret_key = "THIS_IS_A_TESTING_SECRET_KEY"

encodings = {
    "utf8": "UTF-8",
    "utf-16": "UTF-16",
    "latin_1": "Latin-1",
    "mac_roman": "Mac OS Roman",
    "cp1252": "Windows Codepage 1252"
}

# Filters and functions for use within the match template
def flatten_record(record, headers):
    return u"; ".join([u"{0}: {1}".format(key, record.values[key]) for key in headers])

def get_identifier(element="", incoming_line=None, record_id="", key=""):
    if not isinstance(incoming_line, str):
        line = str(incoming_line)
    else:
        line = incoming_line or ""

    return u"{0}{1}".format(element, hashlib.sha512(".".join([line, record_id or "", key or ""])).hexdigest())

def generate_header_mapping(profile):
    header_map = {}

    for mapping in profile.mappings:
        if mapping.key not in header_map:
            header_map[mapping.key] = mapping.master_key

    return header_map

def group_results(results, nresults):
    # Group result lines into tuples (line no, results list) by the original input line.
    # Built a list whose elements are tuples (line no, result list), where the result
    # list is sorted by match score and clamped to the best nresults

    return sorted([(k, sorted(list(g), key=attrgetter("score"), reverse=True)[:nresults])
                   for k, g in groupby(results, lambda res: res.incoming.input_line())],
                  key=itemgetter(0))

@app.route("/")
def start():
    return render_template("index.html",
                           encodings=sorted(encodings.items(), key=itemgetter(1)),
                           input_formats=trapeza.formats.available_input_formats(),
                           output_formats=trapeza.formats.available_output_formats())

@app.route("/run", methods=["POST"])
def step_one():
    try:
        # Load master sheet or processed master file and profile
        master_file = request.files["master"]

        if "input_format" in request.form \
                and request.form.get("input_format") in trapeza.formats.available_input_formats():
            master_format = request.form.get("input_format")
        else:
            master_format = trapeza.get_format(master_file.filename, "csv")

        if "input_encoding" in request.form and request.form.get("input_encoding") in encodings:
            input_encoding = request.form.get("input_encoding")
        else:
            input_encoding = "utf8"

        if master_format == "trapeza":
            # Master file has been preprocessed
            processed_master = pickle.load(master_file)
            master = processed_master.source
            profile = processed_master.profile
        else:
            processed_master = None
            master = trapeza.load_source(request.files["master"], master_format, encoding=input_encoding)
            profile = trapeza.match.Profile(
                source=trapeza.load_source(request.files["profile"],
                                           trapeza.get_format(request.files["profile"].filename, "csv"),
                                           encoding=input_encoding))

        # Load incoming file
        if "input_format" in request.form \
                and request.form.get("input_format") in trapeza.formats.available_input_formats():
            incoming_format = request.form.get("input_format")
        else:
            incoming_format = trapeza.get_format(request.files["incoming"].filename, "csv")

        incoming = trapeza.load_source(request.files["incoming"], incoming_format, encoding=input_encoding)
    except IOError:
        flash("An error occurred during loading of input files. Please provide files in a format that Trapeza "
              "understands and ensure you choose the correct text encoding.")
        return redirect(url_for("start"))

    # Determine primary key and set in Master. We do not set the primary key on incoming or the output.
    # If the user feels like matching multiple input lines to the same master records, that's fine;
    # we don't want to throw an exception.
    try:
        primary_key = request.form["primary_key"]
        master.set_primary_key(primary_key)
    except KeyError:
        flash("One or more master records are missing the specified primary key.")
        return redirect(url_for("start"))

    # Bring in the parameters.
    try:
        cutoff = abs(int(request.form["cutoff"])) or 0
        autocutoff = abs(int(request.form["autocutoff"])) or 0
        nresults = abs(int(request.form["nresults"])) or 5
        display_diff = "display_diff" in request.form
        include_unmatched_records = "include_unmatched_records" in request.form
        output_only_modified_entries = "output_only_modified_entries" in request.form
        include_re_new_address_flag = "include_re_new_address_flag" in request.form
        if request.form.get("line_endings") in ["cr", "crlf", "lf"]:
            line_endings = request.form.get("line_endings")
        else:
            line_endings = "lf"
    except [KeyError, TypeError]:
        flash("An invalid option was specified.")
        return redirect(url_for("start"))

    # Perform the comparison.
    try:
        raw_results = group_results(profile.compare_sources(processed_master or master, incoming, cutoff), nresults)

        # Pull out automatchable results

        automatches = []
        results = []
        sys.stderr.write("Autocutoff enabled. Threshold = {}\n".format(autocutoff))
        if autocutoff != 0:
            for result in raw_results:
                # Each result is a (line number, results list) tuple.
                sys.stderr.write("For input line {}, highest result value = {}\n".format(result[0], result[1][0].score))
                if result[1][0].score >= autocutoff and \
                    (len(result[1]) == 1 or (len(result[1]) > 1 and result[1][1].score < autocutoff)):
                        # We have exactly one match with a score at or above the cutoff.
                        # Automatically match this record.
                        sys.stderr.write("Automatching input line {} to record {} with score {}\n".format(
                             result[0], result[1][0].master.record_id(), result[1][0].score))
                        automatches.append((result[0], result[1][0:1]))
                else:
                        results.append(result)

            sys.stderr.write("Automatched {} records\n".format(len(automatches)))
        else:
            results = raw_results

    except Exception as e:
        flash("An error occurred during matching ({})".format(e))
        return redirect(url_for("start"))

    try:
        # Save the details of the operation
        if request.form.get("output_format") and \
           request.form.get("output_format") in trapeza.formats.available_output_formats():
            output_format = request.form.get("output_format")
        else:
            output_format = "csv"

        if request.form.get("output_encoding") and request.form.get("output_encoding") in encodings:
            output_encoding = request.form.get("output_encoding")
        else:
            output_encoding = "utf8"

        operation = {"master": master,
                     "incoming": incoming,
                     "profile": profile,
                     "primary_key": primary_key,
                     "results": results,
                     "automatches": automatches,
                     "cutoff": cutoff,
                     "output_format": output_format,
                     "output_encoding": output_encoding,
                     "line_endings": line_endings,
                     "display_diff": display_diff,
                     "include_unmatched_records": include_unmatched_records,
                     "output_only_modified_entries": output_only_modified_entries,
                     "include_re_new_address_flag": include_re_new_address_flag}

        outfile = tempfile.NamedTemporaryFile(delete=False)
        session["file"] = outfile.name
        pickle.dump(operation, outfile, pickle.HIGHEST_PROTOCOL)
        outfile.close()
    except Exception as e:
        flash("An error occurred while saving output ({})".format(e))
        return redirect(url_for("start"))

    return render_template("match.html",
                           results=results,
                           primary_key=primary_key,
                           header_map=generate_header_mapping(profile),
                           incoming_headers=incoming.headers(),
                           master_headers=master.headers(),
                           flatten_record=flatten_record,
                           get_identifier=get_identifier,
                           display_diff=display_diff,
                           include_re_new_address_flag=include_re_new_address_flag)

@app.route("/dl", methods=["POST"])
def step_two():
    try:
        with open(session["file"], "rb") as sess_file:
            operation = pickle.load(sess_file)

        os.unlink(session["file"])
    except IOError:
        flash("An error occurred while loading processed results.")
        return redirect(url_for("start"))

    try:
        primary_key = operation["primary_key"]
        header_map = generate_header_mapping(operation["profile"])

        output = trapeza.Source(operation["incoming"].headers())
        output.add_column(primary_key)

        if operation["include_re_new_address_flag"]:
            if "New Address?" not in output.headers():
                output.add_column("New Address?")

        matched_records = []

        for (original_line, matches) in operation["results"]:

            matched_records.append(original_line)

            master_id = request.form.get(get_identifier('select', original_line))

            if master_id or operation["include_unmatched_records"]:
                out_record = trapeza.Record(matches[0].incoming.values)
                out_record.values[primary_key] = master_id or u""
                modified = False

                for key in out_record.values:
                    if key != primary_key:
                        value = request.form.get(get_identifier("select", original_line, master_id, key))

                        if value == "MASTER":
                            if not operation["output_only_modified_entries"]:
                                out_record.values[key] = operation["master"].get_record_with_id(master_id).values[
                                    header_map[key]]
                            else:
                                out_record.values[key] = ""
                        elif value == "USER":
                            user_val = request.form.get(get_identifier("userentrybox", original_line, master_id, key))
                            if value is not None and len(value) > 0:
                                out_record.values[key] = user_val
                                modified = True
                        elif value == "INCOMING" or value == "" or value is None:
                            # If the user made no selection, assume that we're to retain the incoming data.
                            modified = True
                            pass
                        else:
                            raise Exception("Invalid form data.")

                if operation["include_re_new_address_flag"]:
                    new_address = get_identifier('newaddressbox', original_line) in request.form
                    out_record.values["New Address?"] = "TRUE" if modified and new_address else "FALSE"

                output.add_record(out_record)

        # Process records which were automatically matched, if any.

        for (original_line, matches) in operation["automatches"]:
            out_record = trapeza.Record(matches[0].incoming.values)
            out_record.values[primary_key] = matches[0].master.record_id()
            sys.stderr.write("Writing automatched record {} with master ID {}\n"
                             .format(original_line, matches[0].master.record_id()))
            output.add_record(out_record)

        # If we are outputting unmatched records, we also must collect and output any record which didn't have a match
        # over the cutoff (which won't appear in the matched list)

        # Add line identifiers for automatched records to the matched set so we don't get duplicate output.
        matched_records.extend(map(itemgetter(0), operation["automatches"]))

        for record in operation["incoming"].records():
            if record.input_line() not in matched_records:
                out_record = trapeza.Record(record.values)
                out_record.values[primary_key] = u""
                if operation["include_re_new_address_flag"]:
                    out_record.values["New Address?"] = "FALSE"

                output.add_record(record)
    except Exception as e:
        flash("An error occurred while processing matches ({})".format(e))
        return redirect(url_for("start"))

    try:
        out_buffer = io.BytesIO()

        endings = {"crlf": "\r\n", "lf": "\n", "cr": "\r"}

        trapeza.write_source(output, out_buffer, operation["output_format"], encoding=operation["output_encoding"],
                             line_endings=endings.get(operation["line_endings"]))

        data = out_buffer.getvalue()
        out_buffer.close()

        response = make_response(data)
        response.headers["Content-Disposition"] = "attachment; filename=output.{}".format(operation["output_format"])
        response.headers["Content-Type"] = "application/octet-stream"

        return response
    except Exception as e:
        flash("An error occurred while writing output ({})".format(e))
        return redirect(url_for("start"))

if __name__ == "__main__":
    app.debug = True
    app.run()
<commit_msg>Removed logging statements<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
#
#  trapeza-import.py
#
#  Copyright 2013-2014 David Reed <david@ktema.org>
#  This file is available under the terms of the MIT License.
#

from itertools import groupby
from operator import itemgetter, attrgetter

import hashlib
import io
import os
import sys
import pickle
import tempfile
import trapeza
import trapeza.formats
import trapeza.match
from flask import Flask, request, session, make_response, render_template, flash, redirect, url_for

# Accept three Source objects. Run match_incoming_records() on them.
# Render a template showing the user matches and allowing them to select a
# matching record.
# On selection, run diff on the selected records (comparing the columns that are
# mapped by the profile) and offer the user the chance to select one of the two versions
# of each field or to enter their own data.
# On output, generate a copy of the input sheet with a unique ID column added specifying
# the matched record in the database. For each diffed column, if the user did not choose
# to retain existing data, include the user-input value or the value from incoming.
# Maintain all other columns in the original input sheet unaltered for import.

app = Flask(__name__)
app.secret_key = "THIS_IS_A_TESTING_SECRET_KEY"

encodings = {
    "utf8": "UTF-8",
    "utf-16": "UTF-16",
    "latin_1": "Latin-1",
    "mac_roman": "Mac OS Roman",
    "cp1252": "Windows Codepage 1252"
}

# Filters and functions for use within the match template
def flatten_record(record, headers):
    return u"; ".join([u"{0}: {1}".format(key, record.values[key]) for key in headers])

def get_identifier(element="", incoming_line=None, record_id="", key=""):
    if not isinstance(incoming_line, str):
        line = str(incoming_line)
    else:
        line = incoming_line or ""

    return u"{0}{1}".format(element, hashlib.sha512(".".join([line, record_id or "", key or ""])).hexdigest())

def generate_header_mapping(profile):
    header_map = {}

    for mapping in profile.mappings:
        if mapping.key not in header_map:
            header_map[mapping.key] = mapping.master_key

    return header_map

def group_results(results, nresults):
    # Group result lines into tuples (line no, results list) by the original input line.
    # Built a list whose elements are tuples (line no, result list), where the result
    # list is sorted by match score and clamped to the best nresults

    return sorted([(k, sorted(list(g), key=attrgetter("score"), reverse=True)[:nresults])
                   for k, g in groupby(results, lambda res: res.incoming.input_line())],
                  key=itemgetter(0))

@app.route("/")
def start():
    return render_template("index.html",
                           encodings=sorted(encodings.items(), key=itemgetter(1)),
                           input_formats=trapeza.formats.available_input_formats(),
                           output_formats=trapeza.formats.available_output_formats())

@app.route("/run", methods=["POST"])
def step_one():
    try:
        # Load master sheet or processed master file and profile
        master_file = request.files["master"]

        if "input_format" in request.form \
                and request.form.get("input_format") in trapeza.formats.available_input_formats():
            master_format = request.form.get("input_format")
        else:
            master_format = trapeza.get_format(master_file.filename, "csv")

        if "input_encoding" in request.form and request.form.get("input_encoding") in encodings:
            input_encoding = request.form.get("input_encoding")
        else:
            input_encoding = "utf8"

        if master_format == "trapeza":
            # Master file has been preprocessed
            processed_master = pickle.load(master_file)
            master = processed_master.source
            profile = processed_master.profile
        else:
            processed_master = None
            master = trapeza.load_source(request.files["master"], master_format, encoding=input_encoding)
            profile = trapeza.match.Profile(
                source=trapeza.load_source(request.files["profile"],
                                           trapeza.get_format(request.files["profile"].filename, "csv"),
                                           encoding=input_encoding))

        # Load incoming file
        if "input_format" in request.form \
                and request.form.get("input_format") in trapeza.formats.available_input_formats():
            incoming_format = request.form.get("input_format")
        else:
            incoming_format = trapeza.get_format(request.files["incoming"].filename, "csv")

        incoming = trapeza.load_source(request.files["incoming"], incoming_format, encoding=input_encoding)
    except IOError:
        flash("An error occurred during loading of input files. Please provide files in a format that Trapeza "
              "understands and ensure you choose the correct text encoding.")
        return redirect(url_for("start"))

    # Determine primary key and set in Master. We do not set the primary key on incoming or the output.
    # If the user feels like matching multiple input lines to the same master records, that's fine;
    # we don't want to throw an exception.
    try:
        primary_key = request.form["primary_key"]
        master.set_primary_key(primary_key)
    except KeyError:
        flash("One or more master records are missing the specified primary key.")
        return redirect(url_for("start"))

    # Bring in the parameters.
    try:
        cutoff = abs(int(request.form["cutoff"])) or 0
        autocutoff = abs(int(request.form["autocutoff"])) or 0
        nresults = abs(int(request.form["nresults"])) or 5
        display_diff = "display_diff" in request.form
        include_unmatched_records = "include_unmatched_records" in request.form
        output_only_modified_entries = "output_only_modified_entries" in request.form
        include_re_new_address_flag = "include_re_new_address_flag" in request.form
        if request.form.get("line_endings") in ["cr", "crlf", "lf"]:
            line_endings = request.form.get("line_endings")
        else:
            line_endings = "lf"
    except [KeyError, TypeError]:
        flash("An invalid option was specified.")
        return redirect(url_for("start"))

    # Perform the comparison.
    try:
        raw_results = group_results(profile.compare_sources(processed_master or master, incoming, cutoff), nresults)

        # Pull out automatchable results

        automatches = []
        results = []

        if autocutoff != 0:
            for result in raw_results:
                # Each result is a (line number, results list) tuple.
                if result[1][0].score >= autocutoff and \
                    (len(result[1]) == 1 or (len(result[1]) > 1 and result[1][1].score < autocutoff)):
                        # We have exactly one match with a score at or above the cutoff.
                        # Automatically match this record.
                        automatches.append((result[0], result[1][0:1]))
                else:
                        results.append(result)
        else:
            results = raw_results

    except Exception as e:
        flash("An error occurred during matching ({})".format(e))
        return redirect(url_for("start"))

    try:
        # Save the details of the operation
        if request.form.get("output_format") and \
           request.form.get("output_format") in trapeza.formats.available_output_formats():
            output_format = request.form.get("output_format")
        else:
            output_format = "csv"

        if request.form.get("output_encoding") and request.form.get("output_encoding") in encodings:
            output_encoding = request.form.get("output_encoding")
        else:
            output_encoding = "utf8"

        operation = {"master": master,
                     "incoming": incoming,
                     "profile": profile,
                     "primary_key": primary_key,
                     "results": results,
                     "automatches": automatches,
                     "cutoff": cutoff,
                     "output_format": output_format,
                     "output_encoding": output_encoding,
                     "line_endings": line_endings,
                     "display_diff": display_diff,
                     "include_unmatched_records": include_unmatched_records,
                     "output_only_modified_entries": output_only_modified_entries,
                     "include_re_new_address_flag": include_re_new_address_flag}

        outfile = tempfile.NamedTemporaryFile(delete=False)
        session["file"] = outfile.name
        pickle.dump(operation, outfile, pickle.HIGHEST_PROTOCOL)
        outfile.close()
    except Exception as e:
        flash("An error occurred while saving output ({})".format(e))
        return redirect(url_for("start"))

    return render_template("match.html",
                           results=results,
                           primary_key=primary_key,
                           header_map=generate_header_mapping(profile),
                           incoming_headers=incoming.headers(),
                           master_headers=master.headers(),
                           flatten_record=flatten_record,
                           get_identifier=get_identifier,
                           display_diff=display_diff,
                           include_re_new_address_flag=include_re_new_address_flag)

@app.route("/dl", methods=["POST"])
def step_two():
    try:
        with open(session["file"], "rb") as sess_file:
            operation = pickle.load(sess_file)

        os.unlink(session["file"])
    except IOError:
        flash("An error occurred while loading processed results.")
        return redirect(url_for("start"))

    try:
        primary_key = operation["primary_key"]
        header_map = generate_header_mapping(operation["profile"])

        output = trapeza.Source(operation["incoming"].headers())
        output.add_column(primary_key)

        if operation["include_re_new_address_flag"]:
            if "New Address?" not in output.headers():
                output.add_column("New Address?")

        matched_records = []

        for (original_line, matches) in operation["results"]:

            matched_records.append(original_line)

            master_id = request.form.get(get_identifier('select', original_line))

            if master_id or operation["include_unmatched_records"]:
                out_record = trapeza.Record(matches[0].incoming.values)
                out_record.values[primary_key] = master_id or u""
                modified = False

                for key in out_record.values:
                    if key != primary_key:
                        value = request.form.get(get_identifier("select", original_line, master_id, key))

                        if value == "MASTER":
                            if not operation["output_only_modified_entries"]:
                                out_record.values[key] = operation["master"].get_record_with_id(master_id).values[
                                    header_map[key]]
                            else:
                                out_record.values[key] = ""
                        elif value == "USER":
                            user_val = request.form.get(get_identifier("userentrybox", original_line, master_id, key))
                            if value is not None and len(value) > 0:
                                out_record.values[key] = user_val
                                modified = True
                        elif value == "INCOMING" or value == "" or value is None:
                            # If the user made no selection, assume that we're to retain the incoming data.
                            modified = True
                            pass
                        else:
                            raise Exception("Invalid form data.")

                if operation["include_re_new_address_flag"]:
                    new_address = get_identifier('newaddressbox', original_line) in request.form
                    out_record.values["New Address?"] = "TRUE" if modified and new_address else "FALSE"

                output.add_record(out_record)

        # Process records which were automatically matched, if any.

        for (original_line, matches) in operation["automatches"]:
            out_record = trapeza.Record(matches[0].incoming.values)
            out_record.values[primary_key] = matches[0].master.record_id()
            output.add_record(out_record)

        # If we are outputting unmatched records, we also must collect and output any record which didn't have a match
        # over the cutoff (which won't appear in the matched list)

        # Add line identifiers for automatched records to the matched set so we don't get duplicate output.
        matched_records.extend(map(itemgetter(0), operation["automatches"]))

        for record in operation["incoming"].records():
            if record.input_line() not in matched_records:
                out_record = trapeza.Record(record.values)
                out_record.values[primary_key] = u""
                if operation["include_re_new_address_flag"]:
                    out_record.values["New Address?"] = "FALSE"

                output.add_record(record)
    except Exception as e:
        flash("An error occurred while processing matches ({})".format(e))
        return redirect(url_for("start"))

    try:
        out_buffer = io.BytesIO()

        endings = {"crlf": "\r\n", "lf": "\n", "cr": "\r"}

        trapeza.write_source(output, out_buffer, operation["output_format"], encoding=operation["output_encoding"],
                             line_endings=endings.get(operation["line_endings"]))

        data = out_buffer.getvalue()
        out_buffer.close()

        response = make_response(data)
        response.headers["Content-Disposition"] = "attachment; filename=output.{}".format(operation["output_format"])
        response.headers["Content-Type"] = "application/octet-stream"

        return response
    except Exception as e:
        flash("An error occurred while writing output ({})".format(e))
        return redirect(url_for("start"))

if __name__ == "__main__":
    app.debug = True
    app.run()
<|endoftext|>


<commit_before>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
iCallSV.iCallSV -- wrapper to run iCallSV package

iCallSV.iCallSV is a wrapper to the iCallSV package which facilitates calling structural variants from Next Generation Sequencing methods such as Illumina

It defines classes_and_methods

@author:     Ronak H Shah

@copyright:  2015-2016 Ronak H Shah. All rights reserved.

@license:    Apache License 2.0

@contact:    rons.shah@gmail.com

@deffield    updated: Updated
'''

import sys
import os
import time
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
import ConfigParser as configparser
import logging
import make_analysis_dir as mad
import checkparameters as cp
import launch_Run_Delly as lrd
import launch_FilterDellyCalls as lfd
import dellyVcf2Tab as dvcf2tab
import combineVCF as cvcf
import Run_iAnnotateSV as annSV
import dellyVcf2targetSeqView as dvcf2tsv
import Run_targetSeqView as rtsv

__all__ = []
__version__ = 0.1
__date__ = '2015-03-30'
__updated__ = '2015-12-20'

def main(argv=None):  # IGNORE:C0111
    """Command line options."""

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Ronak H Shah on %s.
  Copyright 2015-2016 Ronak H Shah. All rights reserved.

  Licensed under the Apache License 2.0
  http://www.apache.org/licenses/LICENSE-2.0

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    # Setup argument parser
    parser = ArgumentParser(
        description=program_license,
        formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                        default=True, help="set verbosity level [default: %(default)s]")
    parser.add_argument('-V', '--version', action='version', version=program_version_message)
    parser.add_argument(
        "-sc",
        "--svConfig",
        action="store",
        dest="config_file",
        required=True,
        metavar='config.ini',
        help="Full path to the structural variant configuration")
    parser.add_argument(
        "-abam",
        "--caseBam",
        action="store",
        dest="caseBam",
        required=True,
        metavar='caseBAMFile.bam',
        help="Full path to the case bam file")
    parser.add_argument(
        "-bbam",
        "--controlBam",
        action="store",
        dest="controlBam",
        required=True,
        metavar='controlBAMFile.bam',
        help="Full path to the control bam file")
    #parser.add_argument("-afastq", "--caseFastq", action="store", dest="caseFastq", required=False, metavar='caseFastqFile.fastq', help="Full path to the case fastq file")
    #parser.add_argument("-bfastq", "--controlFastq", action="store", dest="controlFastq", required=False, metavar='controlFastqFile.fastq', help="Full path to the control fastq file")
    parser.add_argument(
        "-aId",
        "--caseId",
        action="store",
        dest="caseId",
        required=True,
        metavar='caseID',
        help="Id of the case to be analyzed, this will be the sub-folder")
    parser.add_argument(
        "-bId",
        "--controlId",
        action="store",
        dest="controlId",
        required=True,
        metavar='controlID',
        help="Id of the control to be used, this will be used for filtering variants")
    #parser.add_argument("-t", "--threads", action="store", dest="threads", required=True, metavar='5', help="Number of Threads to be used to run tools")
    parser.add_argument(
        "-o",
        "--outDir",
        action="store",
        dest="outdir",
        required=True,
        metavar='/somepath/output',
        help="Full Path to the output dir.")
    parser.add_argument(
        "-op",
        "--outPrefix",
        action="store",
        dest="outprefix",
        required=True,
        metavar='TumorID',
        help="Id of the Tumor bam file which will be used as the prefix for output files")

    # Process arguments
    args = parser.parse_args()
    #
    # Parse a config ini-style file
    #
    verbose = args.verbose
    # Create Logger if verbose
    loggeroutput = args.outdir + "/" + args.outprefix + "_iCallSV.log"
    logging.basicConfig(
        filename=loggeroutput,
        filemode='w',
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        level=logging.DEBUG)
    # Print if Verbose mode is on
    if(verbose):
        logging.info("iCallSV:Verbose mode on")
    here = os.path.realpath('.')

    config_file = args.config_file
    if(verbose):
        logging.info('iCallSV:Reading configuration from %s', config_file)
    config = configparser.ConfigParser(defaults={'here': here})
    config.read(args.config_file)
    (tag, sampleOutdirForDelly) = mad.makeOutputDir(args, "DellyDir")
    runDellyTag = True
    analysisFiles = []
    analyisisType = ["DEL", "DUP", "INV", "TRA"]
    if(tag == False):
        for analysis in analysisType:
            tag = analysis.lower()
            outputVcf = outputdir + "/" + args.caseId + "_" + tag + ".vcf"
            if(os.path.isfile(outputVcf)):
                tag = True
                runDellyTag = False
                analysisFiles.append(outputVcf)
            else:
                tag = False
    # Decide based on tag what to do
    if(tag):
        if(verbose):
            logging.info(
                'iCallSV:Output of delly for %s will be written in %s',
                args.caseId,
                sampleOutdirForDelly)
        # Run Delly and get the raw calls
        if(runDellyTag):
            (del_vcf, dup_vcf, inv_vcf, tra_vcf) = lrd.launch_delly_for_different_analysis_type(
                args, config, sampleOutdirForDelly)
        else:
            del_vcf, dup_vcf, inv_vcf, tra_vcf = analysisFiles
        # Run Filter Delly and get filtered calls
        (filter_del_vcf,
         filter_dup_vcf,
         filter_inv_vcf,
         filter_tra_vcf) = lfd.launch_filterdellycalls_for_different_analysis_type(args,
                                                                                config,
                                                                                sampleOutdirForDelly,
                                                                                del_vcf,
                                                                                dup_vcf,
                                                                                inv_vcf,
                                                                                tra_vcf)
        # Combine all VCF to a single VCF file
        listOfFilteredVCFfiles = [filter_del_vcf, filter_dup_vcf, filter_inv_vcf, filter_tra_vcf]
        combinedVCF = sampleOutdirForDelly + "/" + args.caseId + "_allSVFiltered.vcf"
        combinedVCF = cvcf.run(listOfFilteredVCFfiles, combinedVCF, verbose)
        #Check if VCF file is empty
        hasRecords = False
        with open(combinedVCF, 'r') as filecontent:
            if any(not line.startswith("#") for line in filecontent):
                hasRecords = True
            else:
                hasRecords = False
        #If there are VCF records do further analysis.
        if(hasRecords):
            combinedAnnVCF = args.caseId + "_allAnnotatedSVFiltered.tab"
            combinedTargetSeqView = args.caseId + "_allSVFiltered_tsvInput.txt"
            combinedTargetSeqViewCscore = args.caseId + "_allSVFiltered_cScore.txt"
            # convert vcf files to tab-delimited using vcf2tab
            combinedTAB = dvcf2tab.vcf2tab(combinedVCF, sampleOutdirForDelly, verbose)
            # Annotate using iAnnotateSV
            combinedAnnTAB = annSV.run(
                config.get(
                    "Python", "PYTHON"), config.get(
                    "iAnnotateSV", "ANNOSV"), config.get(
                    "iAnnotateSV", "GENOMEBUILD"), int(config.get(
                        "iAnnotateSV", "DISTANCE")), config.get(
                            "iAnnotateSV", "CANONICALTRANSCRIPTFILE"), combinedTAB, combinedAnnVCF, sampleOutdirForDelly)
            # convert vcf to targetseqviewformat
            combinedTargetSeqView = dvcf2tsv.Convert2targetSeqView(
                args.caseId,
                args.caseBam,
                args.caseBam,
                combinedVCF,
                sampleOutdirForDelly,
                combinedTargetSeqView)
            # Get Confidence score using targetSeqView
            combinedTargetSeqViewCscore = rtsv.run(config.get("R", "RHOME"),
                                                   config.get("TargetSeqView", "CalculateConfidenceScore"),
                                                   5, args.caseBam, combinedTargetSeqView,
                                                   config.get("TargetSeqView", "GENOMEBUILD"),
                                                   int(config.get("TargetSeqView", "ReadLength")),
                                                   sampleOutdirForDelly, combinedTargetSeqViewCscore)
            # Merge Results from vcf, tab and targetseqview
        else:
            logging.warn("All Records have been filtered in standard filtered step. Thus we will exit the program and not proceed.")
            logging.info("Thank you for using iCallSV.")
            sys.exit(0)
    else:
        if(verbose):
            logging.fatal(
                "The output directory for the %s already exists. Please delete %s folder and rerun",
                args.caseId,
                sampleOutdirForDelly)
            sys.exit(1)

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    totaltime = end_time - start_time
    logging.info("iCallSV:Elapsed time was %g seconds", totaltime)
<commit_msg>Debugging<commit_after>#!/usr/local/bin/python2.7
# encoding: utf-8
'''
iCallSV.iCallSV -- wrapper to run iCallSV package

iCallSV.iCallSV is a wrapper to the iCallSV package which facilitates calling structural variants from Next Generation Sequencing methods such as Illumina

It defines classes_and_methods

@author:     Ronak H Shah

@copyright:  2015-2016 Ronak H Shah. All rights reserved.

@license:    Apache License 2.0

@contact:    rons.shah@gmail.com

@deffield    updated: Updated
'''

import sys
import os
import time
from argparse import ArgumentParser
from argparse import RawDescriptionHelpFormatter
import ConfigParser as configparser
import logging
import make_analysis_dir as mad
import checkparameters as cp
import launch_Run_Delly as lrd
import launch_FilterDellyCalls as lfd
import dellyVcf2Tab as dvcf2tab
import combineVCF as cvcf
import Run_iAnnotateSV as annSV
import dellyVcf2targetSeqView as dvcf2tsv
import Run_targetSeqView as rtsv

__all__ = []
__version__ = 0.1
__date__ = '2015-03-30'
__updated__ = '2015-12-20'

def main(argv=None):  # IGNORE:C0111
    """Command line options."""

    if argv is None:
        argv = sys.argv
    else:
        sys.argv.extend(argv)

    program_name = os.path.basename(sys.argv[0])
    program_version = "v%s" % __version__
    program_build_date = str(__updated__)
    program_version_message = '%%(prog)s %s (%s)' % (program_version, program_build_date)
    program_shortdesc = __import__('__main__').__doc__.split("\n")[1]
    program_license = '''%s

  Created by Ronak H Shah on %s.
  Copyright 2015-2016 Ronak H Shah. All rights reserved.

  Licensed under the Apache License 2.0
  http://www.apache.org/licenses/LICENSE-2.0

  Distributed on an "AS IS" basis without warranties
  or conditions of any kind, either express or implied.

USAGE
''' % (program_shortdesc, str(__date__))

    # Setup argument parser
    parser = ArgumentParser(
        description=program_license,
        formatter_class=RawDescriptionHelpFormatter)
    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true",
                        default=True, help="set verbosity level [default: %(default)s]")
    parser.add_argument('-V', '--version', action='version', version=program_version_message)
    parser.add_argument(
        "-sc",
        "--svConfig",
        action="store",
        dest="config_file",
        required=True,
        metavar='config.ini',
        help="Full path to the structural variant configuration")
    parser.add_argument(
        "-abam",
        "--caseBam",
        action="store",
        dest="caseBam",
        required=True,
        metavar='caseBAMFile.bam',
        help="Full path to the case bam file")
    parser.add_argument(
        "-bbam",
        "--controlBam",
        action="store",
        dest="controlBam",
        required=True,
        metavar='controlBAMFile.bam',
        help="Full path to the control bam file")
    #parser.add_argument("-afastq", "--caseFastq", action="store", dest="caseFastq", required=False, metavar='caseFastqFile.fastq', help="Full path to the case fastq file")
    #parser.add_argument("-bfastq", "--controlFastq", action="store", dest="controlFastq", required=False, metavar='controlFastqFile.fastq', help="Full path to the control fastq file")
    parser.add_argument(
        "-aId",
        "--caseId",
        action="store",
        dest="caseId",
        required=True,
        metavar='caseID',
        help="Id of the case to be analyzed, this will be the sub-folder")
    parser.add_argument(
        "-bId",
        "--controlId",
        action="store",
        dest="controlId",
        required=True,
        metavar='controlID',
        help="Id of the control to be used, this will be used for filtering variants")
    #parser.add_argument("-t", "--threads", action="store", dest="threads", required=True, metavar='5', help="Number of Threads to be used to run tools")
    parser.add_argument(
        "-o",
        "--outDir",
        action="store",
        dest="outdir",
        required=True,
        metavar='/somepath/output',
        help="Full Path to the output dir.")
    parser.add_argument(
        "-op",
        "--outPrefix",
        action="store",
        dest="outprefix",
        required=True,
        metavar='TumorID',
        help="Id of the Tumor bam file which will be used as the prefix for output files")

    # Process arguments
    args = parser.parse_args()
    #
    # Parse a config ini-style file
    #
    verbose = args.verbose
    # Create Logger if verbose
    loggeroutput = args.outdir + "/" + args.outprefix + "_iCallSV.log"
    logging.basicConfig(
        filename=loggeroutput,
        filemode='w',
        format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
        datefmt='%m/%d/%Y %I:%M:%S %p',
        level=logging.DEBUG)
    # Print if Verbose mode is on
    if(verbose):
        logging.info("iCallSV:Verbose mode on")
    here = os.path.realpath('.')

    config_file = args.config_file
    if(verbose):
        logging.info('iCallSV:Reading configuration from %s', config_file)
    config = configparser.ConfigParser(defaults={'here': here})
    config.read(args.config_file)
    (tag, sampleOutdirForDelly) = mad.makeOutputDir(args, "DellyDir")
    runDellyTag = True
    analysisFiles = []
    analysisType = ["DEL", "DUP", "INV", "TRA"]
    if(tag == False):
        for analysis in analysisType:
            tag = analysis.lower()
            outputVcf = outputdir + "/" + args.caseId + "_" + tag + ".vcf"
            if(os.path.isfile(outputVcf)):
                tag = True
                runDellyTag = False
                analysisFiles.append(outputVcf)
            else:
                tag = False
    # Decide based on tag what to do
    if(tag):
        if(verbose):
            logging.info(
                'iCallSV:Output of delly for %s will be written in %s',
                args.caseId,
                sampleOutdirForDelly)
        # Run Delly and get the raw calls
        if(runDellyTag):
            (del_vcf, dup_vcf, inv_vcf, tra_vcf) = lrd.launch_delly_for_different_analysis_type(
                args, config, sampleOutdirForDelly)
        else:
            del_vcf, dup_vcf, inv_vcf, tra_vcf = analysisFiles
        # Run Filter Delly and get filtered calls
        (filter_del_vcf,
         filter_dup_vcf,
         filter_inv_vcf,
         filter_tra_vcf) = lfd.launch_filterdellycalls_for_different_analysis_type(args,
                                                                                config,
                                                                                sampleOutdirForDelly,
                                                                                del_vcf,
                                                                                dup_vcf,
                                                                                inv_vcf,
                                                                                tra_vcf)
        # Combine all VCF to a single VCF file
        listOfFilteredVCFfiles = [filter_del_vcf, filter_dup_vcf, filter_inv_vcf, filter_tra_vcf]
        combinedVCF = sampleOutdirForDelly + "/" + args.caseId + "_allSVFiltered.vcf"
        combinedVCF = cvcf.run(listOfFilteredVCFfiles, combinedVCF, verbose)
        #Check if VCF file is empty
        hasRecords = False
        with open(combinedVCF, 'r') as filecontent:
            if any(not line.startswith("#") for line in filecontent):
                hasRecords = True
            else:
                hasRecords = False
        #If there are VCF records do further analysis.
        if(hasRecords):
            combinedAnnVCF = args.caseId + "_allAnnotatedSVFiltered.tab"
            combinedTargetSeqView = args.caseId + "_allSVFiltered_tsvInput.txt"
            combinedTargetSeqViewCscore = args.caseId + "_allSVFiltered_cScore.txt"
            # convert vcf files to tab-delimited using vcf2tab
            combinedTAB = dvcf2tab.vcf2tab(combinedVCF, sampleOutdirForDelly, verbose)
            # Annotate using iAnnotateSV
            combinedAnnTAB = annSV.run(
                config.get(
                    "Python", "PYTHON"), config.get(
                    "iAnnotateSV", "ANNOSV"), config.get(
                    "iAnnotateSV", "GENOMEBUILD"), int(config.get(
                        "iAnnotateSV", "DISTANCE")), config.get(
                            "iAnnotateSV", "CANONICALTRANSCRIPTFILE"), combinedTAB, combinedAnnVCF, sampleOutdirForDelly)
            # convert vcf to targetseqviewformat
            combinedTargetSeqView = dvcf2tsv.Convert2targetSeqView(
                args.caseId,
                args.caseBam,
                args.caseBam,
                combinedVCF,
                sampleOutdirForDelly,
                combinedTargetSeqView)
            # Get Confidence score using targetSeqView
            combinedTargetSeqViewCscore = rtsv.run(config.get("R", "RHOME"),
                                                   config.get("TargetSeqView", "CalculateConfidenceScore"),
                                                   5, args.caseBam, combinedTargetSeqView,
                                                   config.get("TargetSeqView", "GENOMEBUILD"),
                                                   int(config.get("TargetSeqView", "ReadLength")),
                                                   sampleOutdirForDelly, combinedTargetSeqViewCscore)
            # Merge Results from vcf, tab and targetseqview
        else:
            logging.warn("All Records have been filtered in standard filtered step. Thus we will exit the program and not proceed.")
            logging.info("Thank you for using iCallSV.")
            sys.exit(0)
    else:
        if(verbose):
            logging.fatal(
                "The output directory for the %s already exists. Please delete %s folder and rerun",
                args.caseId,
                sampleOutdirForDelly)
            sys.exit(1)

if __name__ == "__main__":
    start_time = time.time()
    main()
    end_time = time.time()
    totaltime = end_time - start_time
    logging.info("iCallSV:Elapsed time was %g seconds", totaltime)
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import copy
import logging
from collections import OrderedDict

from tldp.utils import max_mtime, mtime_gt

from tldp.sources import SourceCollection
from tldp.outputs import OutputCollection

logger = logging.getLogger(__name__)

# -- any individual document (source or output) will have a status
#    from the following list of status_types
#
stypes = OrderedDict()
stypes['source'] = 'found in source repository'
stypes['output'] = 'found in output repository'
stypes['published'] = 'matching stem in source/output; doc is up to date'
stypes['stale'] = 'matching stem in source/output; but source is newer'
stypes['orphan'] = 'stem located in output, but no source found (i.e. old?)'
stypes['broken'] = 'output is missing an expected output format (e.g. PDF)'
stypes['new'] = 'stem located in source, but missing in output; unpublished'

status_types = stypes.keys()

# -- the user probably doesn't usually care (too much) about listing
#    every single published document and source document, but is probably
#    mostly interested in specific documents grouped by status; so the
#    status_classes are just sets of status_types
#
status_classes = OrderedDict(zip(status_types, [[x] for x in status_types]))
status_classes['outputs'] = ['output']
status_classes['sources'] = ['source']
status_classes['orphans'] = ['orphan']
status_classes['orphaned'] = ['orphan']
status_classes['problems'] = ['orphan', 'broken', 'stale']
status_classes['work'] = ['new', 'orphan', 'broken', 'stale']
status_classes['all'] = ['published', 'new', 'orphan', 'broken', 'stale']

class Inventory(object):
    '''a container for classifying documents by their status

    Every SourceDocument has no more than one matching OutputDirectory.

    The Inventory class encodes the logic for identifying the following
    different status possibilities for an arbitrary set of SourceDocuments and
    OutputDirectorys.

    The following are possible values for status:
       - 'source':  a source document before any status detection
       - 'output':  an output document before any status detection
       - 'new':  a source document without any matching output stem
       - 'published':  a pair of source/output documents with matching stems
       - 'orphan':  an output document without any matching source stem
       - 'broken':  a published document with missing output files
       - 'stale':  a published document with new(er) source files

    The Inventory object is intended to be used to identify work that needs to
    be done on individual source documents to produce up-to-date output
    documents.
    '''
    def __repr__(self):
        return '<%s: %d published, %d orphan, %d new, %d stale, %d broken>' % (
               self.__class__.__name__,
               len(self.published),
               len(self.orphan),
               len(self.new),
               len(self.stale),
               len(self.broken),
               )

    def __init__(self, pubdir, sourcedirs):
        '''construct an Inventory

        pubdir: path to the OutputCollection

        sourcedirs: a list of directories which could be passed to the
          SourceCollection object; essentially a directory containing
          SourceDocuments; for example LDP/LDP/howto/linuxdoc and
          LDP/LDP/guide/docbook
        '''
        self.output = OutputCollection(pubdir)
        self.source = SourceCollection(sourcedirs)
        s = copy.deepcopy(self.source)
        o = copy.deepcopy(self.output)
        sset = set(s.keys())
        oset = set(o.keys())

        # -- orphan identification
        #
        self.orphan = OutputCollection()
        for doc in oset.difference(sset):
            self.orphan[doc] = o[doc]
            del o[doc]
            self.orphan[doc].status = 'orphan'
        logger.debug("Identified %d orphan documents: %r.", len(self.orphan),
                     self.orphan.keys())

        # -- unpublished ('new') identification
        #
        self.new = SourceCollection()
        for doc in sset.difference(oset):
            self.new[doc] = s[doc]
            del s[doc]
            self.new[doc].status = 'new'
        logger.debug("Identified %d new documents: %r.", len(self.new),
                     self.new.keys())

        # -- published identification; source and output should be same size
        assert len(s) == len(o)
        for stem, odoc in o.items():
            sdoc = s[stem]
            sdoc.output = odoc
            odoc.source = sdoc
            sdoc.status = sdoc.output.status = 'published'
        self.published = s
        logger.debug("Identified %d published documents.", len(self.published))

        # -- stale identification
        #
        self.stale = SourceCollection()
        for stem, sdoc in s.items():
            odoc = sdoc.output
            mtime = max_mtime(odoc.statinfo)
            fset = mtime_gt(mtime, sdoc.statinfo)
            if fset:
                sdoc.newer = fset
                for f in fset:
                    logger.debug("%s found updated source file %s", stem, f)
                odoc.status = sdoc.status = 'stale'
                self.stale[stem] = sdoc
        logger.debug("Identified %d stale documents: %r.", len(self.stale),
                     self.stale.keys())

        # -- stale identification
        #
        self.broken = SourceCollection()
        for stem, sdoc in s.items():
            if not sdoc.output.iscomplete:
                self.broken[stem] = sdoc
                sdoc.status = sdoc.output.status = 'broken'
        logger.debug("Identified %d broken documents: %r.", len(self.broken),
                     self.broken.keys())

    def getByStatusClass(self, status_class):
        desired = status_classes.get(status_class, None)
        assert isinstance(desired, list)
        collection = SourceCollection()
        for status_type in desired:
            collection.update(getattr(self, status_type))
        return collection

    @property
    def outputs(self):
        return self.getByStatusClass('outputs')

    @property
    def sources(self):
        return self.getByStatusClass('sources')

    @property
    def problems(self):
        return self.getByStatusClass('problems')

    @property
    def work(self):
        return self.getByStatusClass('work')

    @property
    def orphans(self):
        return self.getByStatusClass('orphans')

    @property
    def orphaned(self):
        return self.getByStatusClass('orphaned')

    @property
    def all(self):
        return self.getByStatusClass('all')

#
# -- end of file
<commit_msg>calculate stale by MD5s; swap stale/broken<commit_after>#! /usr/bin/python
# -*- coding: utf8 -*-

from __future__ import absolute_import, division, print_function
from __future__ import unicode_literals

import copy
import logging
from collections import OrderedDict

from tldp.utils import max_mtime, mtime_gt

from tldp.sources import SourceCollection
from tldp.outputs import OutputCollection

logger = logging.getLogger(__name__)

# -- any individual document (source or output) will have a status
#    from the following list of status_types
#
stypes = OrderedDict()
stypes['source'] = 'found in source repository'
stypes['output'] = 'found in output repository'
stypes['published'] = 'matching stem in source/output; doc is up to date'
stypes['stale'] = 'matching stem in source/output; but source is newer'
stypes['orphan'] = 'stem located in output, but no source found (i.e. old?)'
stypes['broken'] = 'output is missing an expected output format (e.g. PDF)'
stypes['new'] = 'stem located in source, but missing in output; unpublished'

status_types = stypes.keys()

# -- the user probably doesn't usually care (too much) about listing
#    every single published document and source document, but is probably
#    mostly interested in specific documents grouped by status; so the
#    status_classes are just sets of status_types
#
status_classes = OrderedDict(zip(status_types, [[x] for x in status_types]))
status_classes['outputs'] = ['output']
status_classes['sources'] = ['source']
status_classes['orphans'] = ['orphan']
status_classes['orphaned'] = ['orphan']
status_classes['problems'] = ['orphan', 'broken', 'stale']
status_classes['work'] = ['new', 'orphan', 'broken', 'stale']
status_classes['all'] = ['published', 'new', 'orphan', 'broken', 'stale']

class Inventory(object):
    '''a container for classifying documents by their status

    Every SourceDocument has no more than one matching OutputDirectory.

    The Inventory class encodes the logic for identifying the following
    different status possibilities for an arbitrary set of SourceDocuments and
    OutputDirectorys.

    The following are possible values for status:
       - 'source':  a source document before any status detection
       - 'output':  an output document before any status detection
       - 'new':  a source document without any matching output stem
       - 'published':  a pair of source/output documents with matching stems
       - 'orphan':  an output document without any matching source stem
       - 'broken':  a published document with missing output files
       - 'stale':  a published document with new(er) source files

    The Inventory object is intended to be used to identify work that needs to
    be done on individual source documents to produce up-to-date output
    documents.
    '''
    def __repr__(self):
        return '<%s: %d published, %d orphan, %d new, %d stale, %d broken>' % (
               self.__class__.__name__,
               len(self.published),
               len(self.orphan),
               len(self.new),
               len(self.stale),
               len(self.broken),
               )

    def __init__(self, pubdir, sourcedirs):
        '''construct an Inventory

        pubdir: path to the OutputCollection

        sourcedirs: a list of directories which could be passed to the
          SourceCollection object; essentially a directory containing
          SourceDocuments; for example LDP/LDP/howto/linuxdoc and
          LDP/LDP/guide/docbook
        '''
        self.output = OutputCollection(pubdir)
        self.source = SourceCollection(sourcedirs)
        s = copy.deepcopy(self.source)
        o = copy.deepcopy(self.output)
        sset = set(s.keys())
        oset = set(o.keys())

        # -- orphan identification
        #
        self.orphan = OutputCollection()
        for doc in oset.difference(sset):
            self.orphan[doc] = o[doc]
            del o[doc]
            self.orphan[doc].status = 'orphan'
        logger.debug("Identified %d orphan documents: %r.", len(self.orphan),
                     self.orphan.keys())

        # -- unpublished ('new') identification
        #
        self.new = SourceCollection()
        for doc in sset.difference(oset):
            self.new[doc] = s[doc]
            del s[doc]
            self.new[doc].status = 'new'
        logger.debug("Identified %d new documents: %r.", len(self.new),
                     self.new.keys())

        # -- published identification; source and output should be same size
        assert len(s) == len(o)
        for stem, odoc in o.items():
            sdoc = s[stem]
            sdoc.output = odoc
            odoc.source = sdoc
            sdoc.status = sdoc.output.status = 'published'
        self.published = s
        logger.debug("Identified %d published documents.", len(self.published))

        # -- broken identification
        #
        self.broken = SourceCollection()
        for stem, sdoc in s.items():
            if not sdoc.output.iscomplete:
                self.broken[stem] = sdoc
                sdoc.status = sdoc.output.status = 'broken'
        logger.debug("Identified %d broken documents: %r.", len(self.broken),
                     self.broken.keys())

        # -- stale identification
        #
        self.stale = SourceCollection()
        for stem, sdoc in s.items():
            odoc = sdoc.output
            omd5, smd5 = odoc.md5sums, sdoc.md5sums
            if omd5 != smd5:
                logger.debug("%s differing MD5 sets %r %r", stem, smd5, omd5)
                changed = set()
                for gone in set(omd5.keys()).difference(smd5.keys()):
                    logger.debug("%s gone %s", stem, gone)
                    changed.add(('gone', gone))
                for new in set(smd5.keys()).difference(omd5.keys()):
                    changed.add(('new', new))
                for sfn in set(smd5.keys()).intersection(omd5.keys()):
                    if smd5[sfn] != omd5[sfn]:
                        changed.add(('changed', sfn))
                for why, sfn in changed:
                    logger.debug("%s differing source %s (%s)", stem, sfn, why)
                odoc.status = sdoc.status = 'stale'
                sdoc.differing = changed
                self.stale[stem] = sdoc
        logger.debug("Identified %d stale documents: %r.", len(self.stale),
                     self.stale.keys())

    def getByStatusClass(self, status_class):
        desired = status_classes.get(status_class, None)
        assert isinstance(desired, list)
        collection = SourceCollection()
        for status_type in desired:
            collection.update(getattr(self, status_type))
        return collection

    @property
    def outputs(self):
        return self.getByStatusClass('outputs')

    @property
    def sources(self):
        return self.getByStatusClass('sources')

    @property
    def problems(self):
        return self.getByStatusClass('problems')

    @property
    def work(self):
        return self.getByStatusClass('work')

    @property
    def orphans(self):
        return self.getByStatusClass('orphans')

    @property
    def orphaned(self):
        return self.getByStatusClass('orphaned')

    @property
    def all(self):
        return self.getByStatusClass('all')

#
# -- end of file
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ResPartner(orm.Model):
    """ Model name: ResPartner
    """

    _inherit = 'res.partner'

    def _get_partner_duelist_check(self, cr, uid, ids, fields, args,
            context=None):
        ''' Fields function for calculate
        '''
        res = {}
        now = datetime.now().strftime(DEFAULT_SERVER_DATE_FORMAT)

        for partner in self.browse(
                cr, uid, ids, context=context):
            res[partner.id] = {}
            res[partner.id]['deadline_present'] = False
            res[partner.id]['deadline_comment'] = False

            # This partner payment deadline:
            for payment in partner.duelist_ids:
                if payment.deadline < now:
                    res[partner.id]['deadline_present'] = True
                    res[partner.id]['deadline_comment'] = _(
                        'PAGAMENTI SCADUTI!')
                    break

            # Parent partner payment deadline:
            if partner.docnaet_parent_id:
                for payment in partner.docnaet_parent_id.duelist_ids:
                    if payment.deadline < now:
                        res[partner.id]['deadline_present'] = True
                        res[partner.id]['deadline_comment'] = _(
                            'PAGAMENTI SCADUTI MASTER PARTNER!')
                        break
        return res

    _columns = {
        'deadline_present': fields.function(
            _get_partner_duelist_check, method=True,
            type='boolean', string='Deadline present',
            store=False, multi=True),
        'deadline_comment': fields.function(
            _get_partner_duelist_check, method=True,
            type='char', size=80, string='Deadline comment',
            store=False, multi=True),
        }

class DocnaetDocument(orm.Model):
    """ Model name: Docnaet Document
    """
    _inherit = 'docnaet.document'

    _columns = {
        'deadline_present': fields.related(
            'partner_id', 'deadline_present',
            type='boolean', string='Deadline present',
            store=False),
        'deadline_comment': fields.related(
            'partner_id', 'deadline_comment',
            type='char', string='Deadline comment',
            store=False),
        'duelist_ids': fields.related(
            'partner_id', 'duelist_ids',
            type='one2many', relation='sql.payment.duelist',
            string='Duelist'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<commit_msg>correct definition for open payment<commit_after># -*- coding: utf-8 -*-
###############################################################################
#
#    Copyright (C) 2001-2014 Micronaet SRL (<http://www.micronaet.it>).
#
#    This program is free software: you can redistribute it and/or modify
#    it under the terms of the GNU Affero General Public License as published
#    by the Free Software Foundation, either version 3 of the License, or
#    (at your option) any later version.
#
#    This program is distributed in the hope that it will be useful,
#    but WITHOUT ANY WARRANTY; without even the implied warranty of
#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#    GNU Affero General Public License for more details.
#
#    You should have received a copy of the GNU Affero General Public License
#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################
import os
import sys
import logging
import openerp
import openerp.netsvc as netsvc
import openerp.addons.decimal_precision as dp
from openerp.osv import fields, osv, expression, orm
from datetime import datetime, timedelta
from dateutil.relativedelta import relativedelta
from openerp import SUPERUSER_ID
from openerp import tools
from openerp.tools.translate import _
from openerp.tools.float_utils import float_round as round
from openerp.tools import (DEFAULT_SERVER_DATE_FORMAT,
    DEFAULT_SERVER_DATETIME_FORMAT,
    DATETIME_FORMATS_MAP,
    float_compare)

_logger = logging.getLogger(__name__)

class ResPartner(orm.Model):
    """ Model name: ResPartner
    """

    _inherit = 'res.partner'

    def _get_partner_duelist_check(self, cr, uid, ids, fields, args,
            context=None):
        ''' Fields function for calculate
        '''
        res = {}
        now = datetime.now().strftime(DEFAULT_SERVER_DATE_FORMAT)

        for partner in self.browse(
                cr, uid, ids, context=context):
            res[partner.id] = {}
            res[partner.id]['deadline_present'] = False
            res[partner.id]['deadline_comment'] = False

            # This partner payment deadline:
            for payment in partner.duelist_ids:
                if payment.deadline < now and payment.total > 0:
                    res[partner.id]['deadline_present'] = True
                    res[partner.id]['deadline_comment'] = _(
                        'PAGAMENTI SCADUTI!')
                    break

            # Parent partner payment deadline:
            if partner.docnaet_parent_id:
                for payment in partner.docnaet_parent_id.duelist_ids:
                    if payment.deadline < now:
                        res[partner.id]['deadline_present'] = True
                        res[partner.id]['deadline_comment'] = _(
                            'PAGAMENTI SCADUTI MASTER PARTNER!')
                        break
        return res

    _columns = {
        'deadline_present': fields.function(
            _get_partner_duelist_check, method=True,
            type='boolean', string='Deadline present',
            store=False, multi=True),
        'deadline_comment': fields.function(
            _get_partner_duelist_check, method=True,
            type='char', size=80, string='Deadline comment',
            store=False, multi=True),
        }

class DocnaetDocument(orm.Model):
    """ Model name: Docnaet Document
    """
    _inherit = 'docnaet.document'

    _columns = {
        'deadline_present': fields.related(
            'partner_id', 'deadline_present',
            type='boolean', string='Deadline present',
            store=False),
        'deadline_comment': fields.related(
            'partner_id', 'deadline_comment',
            type='char', string='Deadline comment',
            store=False),
        'duelist_ids': fields.related(
            'partner_id', 'duelist_ids',
            type='one2many', relation='sql.payment.duelist',
            string='Duelist'),
        }
# vim:expandtab:smartindent:tabstop=4:softtabstop=4:shiftwidth=4:
<|endoftext|>