<commit_before># -*- coding: utf-8 -*-

import unittest
from datetime import datetime
import pandas as pd
try:
    from unittest.mock import patch
except ImportError:
    from mock import patch
from pandas.util.testing import assert_frame_equal
from WindAdapter.api import factor_load
from WindAdapter.enums import OutputFormat

class MockWindData(object):
    def __init__(self, data, codes, error_code, fields, times):
        self.Data = data
        self.Codes = codes
        self.ErrorCode = error_code
        self.Fields = fields
        self.Times = times

class TestApi(unittest.TestCase):
    @patch('WindAdapter.data_provider.WindDataProvider.query_data')
    def testFactorLoad_case1(self, mock_query_data):
        start_date = '2016-01-01'
        end_date = '2016-02-01'
        factor_name = 'PB'
        sec_id = ['000001.SZ', '000002.SZ', '000003.SZ']

        mock_data = MockWindData(data=[[1, 3, 5]],
                                 codes=sec_id,
                                 error_code=0,
                                 fields=['PB'],
                                 times=[datetime(2016, 1, 1),
                                        datetime(2016, 2, 1)])
        mock_query_data.return_value = mock_data

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 output_data_format=OutputFormat.MULTI_INDEX_DF,
                                 is_index=False)

        expected = pd.DataFrame(data=[1, 3, 5, 1, 3, 5],
                                index=pd.MultiIndex.from_product([['2016-01-29', '2016-02-01'],
                                                                  ['000001.SZ', '000002.SZ', '000003.SZ']],
                                                                 names=['date', 'secID']),
                                columns=['factor'])
        assert_frame_equal(calculated, expected)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 output_data_format=OutputFormat.PITVOT_TABLE_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[[1, 3, 5], [1, 3, 5]],
                                columns=['000001.SZ', '000002.SZ', '000003.SZ'],
                                index=['2016-01-29', '2016-02-01'])
        assert_frame_equal(calculated, expected)

    @patch('WindAdapter.data_provider.WindDataProvider.query_data')
    @patch('WindAdapter.data_provider.WindDataProvider.biz_days_list')
    @patch('WindAdapter.data_provider.WindDataProvider.advance_date')
    def testFactorLoad_case2(self, mock_adv_date, mock_days_list, mock_query_data):
        start_date = '2016-01-01'
        end_date = '2016-02-01'
        factor_name = 'STDQ'
        sec_id = ['000001.SZ', '000002.SZ', '000003.SZ', '000004.SZ']

        mock_data = MockWindData(data=[[1, 2, 3, 4]],
                                 codes=sec_id,
                                 error_code=0,
                                 fields=['STDQ'],
                                 times=[datetime(2016, 1, 1)])
        mock_query_data.return_value = mock_data
        mock_days_list.return_value = [datetime(2016, 1, 1), datetime(2016, 2, 1)]
        mock_adv_date.return_value = datetime(2015, 9, 1)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 tenor='1Q',
                                 output_data_format=OutputFormat.MULTI_INDEX_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[1, 2, 3, 4, 1, 2, 3, 4],
                                index=pd.MultiIndex.from_product([['2016-01-01', '2016-02-01'],
                                                                  ['000001.SZ', '000002.SZ', '000003.SZ',
                                                                   '000004.SZ']],
                                                                 names=['date', 'secID']),
                                columns=['factor'])
        assert_frame_equal(calculated, expected)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 tenor='1Q',
                                 output_data_format=OutputFormat.PITVOT_TABLE_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[[1, 2, 3, 4], [1, 2, 3, 4]],
                                index=['2016-01-01', '2016-02-01'],
                                columns=['000001.SZ', '000002.SZ', '000003.SZ', '000004.SZ'])
        assert_frame_equal(calculated, expected)
<commit_msg>Fix test<commit_after># -*- coding: utf-8 -*-

import unittest
from datetime import datetime
import pandas as pd

try:
    from unittest.mock import patch
except ImportError:
    from mock import patch
from pandas.util.testing import assert_frame_equal
from WindAdapter.api import factor_load

class MockWindData(object):
    def __init__(self, data, codes, error_code, fields, times):
        self.Data = data
        self.Codes = codes
        self.ErrorCode = error_code
        self.Fields = fields
        self.Times = times

class TestApi(unittest.TestCase):
    @patch('WindAdapter.data_provider.WindDataProvider.query_data')
    def testFactorLoad_case1(self, mock_query_data):
        from WindAdapter.enums import OutputFormat
        start_date = '2016-01-01'
        end_date = '2016-02-01'
        factor_name = 'PB'
        sec_id = ['000001.SZ', '000002.SZ', '000003.SZ']

        mock_data = MockWindData(data=[[1, 3, 5]],
                                 codes=sec_id,
                                 error_code=0,
                                 fields=['PB'],
                                 times=[datetime(2016, 1, 1),
                                        datetime(2016, 2, 1)])
        mock_query_data.return_value = mock_data

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 output_data_format=OutputFormat.MULTI_INDEX_DF,
                                 is_index=False)

        expected = pd.DataFrame(data=[1, 3, 5, 1, 3, 5],
                                index=pd.MultiIndex.from_product([['2016-01-29', '2016-02-01'],
                                                                  ['000001.SZ', '000002.SZ', '000003.SZ']],
                                                                 names=['date', 'secID']),
                                columns=['factor'])
        assert_frame_equal(calculated, expected)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 output_data_format=OutputFormat.PITVOT_TABLE_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[[1, 3, 5], [1, 3, 5]],
                                columns=['000001.SZ', '000002.SZ', '000003.SZ'],
                                index=['2016-01-29', '2016-02-01'])
        assert_frame_equal(calculated, expected)

    @patch('WindAdapter.data_provider.WindDataProvider.query_data')
    @patch('WindAdapter.data_provider.WindDataProvider.biz_days_list')
    @patch('WindAdapter.data_provider.WindDataProvider.advance_date')
    def testFactorLoad_case2(self, mock_adv_date, mock_days_list, mock_query_data):
        from WindAdapter.enums import OutputFormat
        start_date = '2016-01-01'
        end_date = '2016-02-01'
        factor_name = 'STDQ'
        sec_id = ['000001.SZ', '000002.SZ', '000003.SZ', '000004.SZ']

        mock_data = MockWindData(data=[[1, 2, 3, 4]],
                                 codes=sec_id,
                                 error_code=0,
                                 fields=['STDQ'],
                                 times=[datetime(2016, 1, 1)])
        mock_query_data.return_value = mock_data
        mock_days_list.return_value = [datetime(2016, 1, 1), datetime(2016, 2, 1)]
        mock_adv_date.return_value = datetime(2015, 9, 1)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 tenor='1Q',
                                 output_data_format=OutputFormat.MULTI_INDEX_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[1, 2, 3, 4, 1, 2, 3, 4],
                                index=pd.MultiIndex.from_product([['2016-01-01', '2016-02-01'],
                                                                  ['000001.SZ', '000002.SZ', '000003.SZ',
                                                                   '000004.SZ']],
                                                                 names=['date', 'secID']),
                                columns=['factor'])
        assert_frame_equal(calculated, expected)

        calculated = factor_load(start_date=start_date,
                                 end_date=end_date,
                                 factor_name=factor_name,
                                 sec_id=sec_id,
                                 tenor='1Q',
                                 output_data_format=OutputFormat.PITVOT_TABLE_DF,
                                 is_index=False)
        expected = pd.DataFrame(data=[[1, 2, 3, 4], [1, 2, 3, 4]],
                                index=['2016-01-01', '2016-02-01'],
                                columns=['000001.SZ', '000002.SZ', '000003.SZ', '000004.SZ'])
        assert_frame_equal(calculated, expected)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# - coding: utf-8 -
# Copyright (C) 2010 Toms Bauģis <toms.baugis at gmail.com>

import colorsys

import gtk
from lib import graphics
from lib.pytweener import Easing
from random import randint

class EasingBox(graphics.Rectangle):
    def __init__(self, name, x, y, easing_method):
        graphics.Rectangle.__init__(self, 40, 40, 3, fill = "#aaa")
        self.name = name
        self.x = x
        self.y = y
        self.easing_method = easing_method
        self.left_side = True
        self.interactive = True

class Scene(graphics.Scene):
    def __init__(self):
        graphics.Scene.__init__(self)

        self.boxes = []

        classes = Easing()
        for i, easing_class in enumerate(dir(Easing)):
            if easing_class.startswith("__") == False:
                the_class = classes.__getattribute__(easing_class)

                label = graphics.Label(easing_class, color = "#333", x = 10, y = i * 49 + 40)
                self.add_child(label)

                box = EasingBox(easing_class, 90, i * 49 + 30, the_class)
                self.add_child(box)
                self.boxes.append(box)

                label = graphics.Label(easing_class, color = "#333", x = 350, y = i * 49 + 40)
                self.add_child(label)

        self.connect("on-click", self.on_click)

    def on_click(self, area, event, targets):
        if not targets:
            return

        clicked = targets[0]
        easing = clicked.easing_method()

        if clicked.left_side:
            self.animate(clicked, x = 300, easing = easing.__getattribute__("ease_out"))
        else:
            self.animate(clicked, x = 90, easing = easing.__getattribute__("ease_in"))

        clicked.left_side = not clicked.left_side

class BasicWindow:
    def __init__(self):
        window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        window.set_size_request(450, 630)
        window.connect("delete_event", lambda *args: gtk.main_quit())
        window.add(Scene())
        window.show_all()

if __name__ == "__main__":
    example = BasicWindow()
    gtk.main()
<commit_msg>tweening also the color<commit_after>#!/usr/bin/env python
# - coding: utf-8 -
# Copyright (C) 2010 Toms Bauģis <toms.baugis at gmail.com>

import colorsys

import gtk
from lib import graphics
from lib.pytweener import Easing
from random import randint
import datetime as dt

class EasingBox(graphics.Rectangle):
    def __init__(self, name, x, y, easing_method):
        graphics.Rectangle.__init__(self, 40, 40, 3, fill = "#aaa")
        self.name = name
        self.x = x
        self.y = y
        self.easing_method = easing_method
        self.left_side = True
        self.interactive = True

class Scene(graphics.Scene):
    def __init__(self):
        graphics.Scene.__init__(self)

        self.boxes = []

        classes = Easing()
        for i, easing_class in enumerate(dir(Easing)):
            if easing_class.startswith("__") == False:
                the_class = classes.__getattribute__(easing_class)

                label = graphics.Label(easing_class, color = "#333", x = 10, y = i * 49 + 40)
                self.add_child(label)

                box = EasingBox(easing_class, 90, i * 49 + 30, the_class)
                self.add_child(box)
                self.boxes.append(box)

                label = graphics.Label(easing_class, color = "#333", x = 350, y = i * 49 + 40)
                self.add_child(label)

        self.connect("on-click", self.on_click)

    def on_click(self, area, event, targets):
        if not targets:
            return

        clicked = targets[0]
        easing = clicked.easing_method()

        if clicked.left_side:
            self.animate(clicked, x = 300, easing = easing.__getattribute__("ease_out"), fill="#fff000")
        else:
            self.animate(clicked, x = 90, easing = easing.__getattribute__("ease_in"), fill="#aaa")

        clicked.left_side = not clicked.left_side

class BasicWindow:
    def __init__(self):
        window = gtk.Window(gtk.WINDOW_TOPLEVEL)
        window.set_size_request(450, 630)
        window.connect("delete_event", lambda *args: gtk.main_quit())
        window.add(Scene())
        window.show_all()

if __name__ == "__main__":
    example = BasicWindow()
    gtk.main()
<|endoftext|>


<commit_before># This is very scratchy and supports only limited portion of Python functions.

import ast
import math
import inspect

class LatexifyVisitor(ast.NodeVisitor):
  def generic_visit(self, node):
    return str(node)

  def visit_Module(self, node):
    return self.visit(node.body[0])

  def visit_FunctionDef(self, node):
    name_str = r'\mathrm{' + str(node.name) + '}'
    arg_strs = [str(arg.arg) for arg in node.args.args]
    body_str = self.visit(node.body[0])
    return name_str + '(' + ', '.join(arg_strs) + r') \triangleq ' + body_str

  def visit_Return(self, node):
    return self.visit(node.value)

  def visit_Call(self, node):
    builtin_callees = {
        'math.acos': (r'\arccos{\left(', r'\right)}'),
        'math.acosh': (r'\operatorname{arccosh}{\left(', r'\right)}'),
        'math.asin': (r'\arcsin{\left(', r'\right)}'),
        'math.asinh': (r'\operatorname{arcsinh}{\left(', r'\right)}'),
        'math.atan': (r'\arctan{\left(', r'\right)}'),
        'math.atanh': (r'\operatorname{arctanh}{\left(', r'\right)}'),
        'math.ceil': (r'\left\lceil', r'\right\rceil'),
        'math.cos': (r'\cos{\left(', r'\right)}'),
        'math.cosh': (r'\cosh{\left(', r'\right)}'),
        'math.exp': (r'\exp{\left(', r'\right)'),
        'math.fabs': (r'\left|', r'\right|'),
        'math.factorial': (r'\left(', r'\right)!'),
        'math.floor': (r'\left\lfloor', r'\right\rfloor'),
        'math.fsum': (r'\sum \left(', r'\right)'),
        'math.gamma': (r'\Gamma \left(', r'\right)'),
        'math.log': (r'\log{\left(', r'\right)}'),
        'math.log10': (r'\log_{10}{\left(', r'\right)'),
        'math.log2': (r'\log_{2}{\left(', r'\right)'),
        'math.prod': (r'\prod \left(', r'\right)'),
        'math.sin': (r'\sin{\left(', r'\right)}'),
        'math.sinh': (r'\sinh{\left(', r'\right)}'),
        'math.sqrt': (r'\sqrt{', '}'),
        'math.tan': (r'\tan{\left(', r'\right)}'),
        'math.tanh': (r'\tanh{\left(', r'\right)}'),
        'sum': (r'\sum \left(', r'\right)'),
    }

    callee_str = self.visit(node.func)
    if callee_str in builtin_callees:
      lstr, rstr = builtin_callees[callee_str]
    else:
      if callee_str.startswith('math.'):
        callee_str = callee_str[5:]
      lstr = r'\operatorname{' + callee_str + '}\left('
      rstr = r'\right)'

    arg_strs = [self.visit(arg) for arg in node.args]
    return lstr + ', '.join(arg_strs) + rstr

  def visit_Attribute(self, node):
    vstr = self.visit(node.value)
    astr = str(node.attr)
    return vstr + '.' + astr

  def visit_Name(self, node):
    return str(node.id)

  def visit_Num(self, node):
    return str(node.n)

  def visit_UnaryOp(self, node):
    def _wrap(child):
      latex = self.visit(child)
      if isinstance(child, ast.BinOp) and isinstance(child.op, (ast.Add, ast.Sub)):
        return '(' + latex + ')'
      return latex

    reprs = {
        ast.UAdd: (lambda: _wrap(node.operand)),
        ast.USub: (lambda: '-' + _wrap(node.operand)),
    }

    if type(node.op) in reprs:
      return reprs[type(node.op)]()
    else:
      return r'\mathrm{unknown_uniop}(' + vstr + ')'

  def visit_BinOp(self, node):
    priority = {
        ast.Add: 10,
        ast.Sub: 10,
        ast.Mult: 20,
        ast.Div: 20,
        ast.Pow: 30,
    }

    def _unwrap(child):
      return self.visit(child)

    def _wrap(child):
      latex = _unwrap(child)
      if isinstance(child, ast.BinOp):
        cp = priority[type(child.op)] if type(child.op) in priority else 100
        pp = priority[type(node.op)] if type(node.op) in priority else 100
        if cp < pp:
          return '(' + latex + ')'
      return latex

    l = node.left
    r = node.right
    reprs = {
        ast.Add: (lambda: _wrap(l) + ' + ' + _wrap(r)),
        ast.Sub: (lambda: _wrap(l) + ' - ' + _wrap(r)),
        ast.Mult: (lambda: _wrap(l) + _wrap(r)),
        ast.Div: (lambda: r'\frac{' + _unwrap(l) + '}{' + _unwrap(r) + '}'),
        ast.Pow: (lambda: _wrap(l) + '^{' + _unwrap(r) + '}'),
    }

    if type(node.op) in reprs:
      return reprs[type(node.op)]()
    else:
      return r'\mathrm{unknown_binop}(' + lstr + ', ' + rstr + ')'

  def visit_Compare(self, node):
    lstr = self.visit(node.left)
    rstr = self.visit(node.comparators[0])

    if isinstance(node.ops[0], ast.Eq):
      return lstr + '=' + rstr
    else:
      return r'\mathrm{unknown_compop}(' + lstr + ', ' + rstr + ')'

  def visit_If(self, node):
    cond_str = self.visit(node.test)
    tstr = self.visit(node.body[0])
    fstr = self.visit(node.orelse[0])
    return r'\left\{ \begin{array}{ll} ' + tstr + r', & \mathrm{if} \ ' + cond_str + r' \\ ' + fstr + r', & \mathrm{otherwise} \end{array} \right.'

def get_latex(fn):
  return LatexifyVisitor().visit(ast.parse(inspect.getsource(fn)))

def with_latex(fn):

  class _LatexifiedFunction:
    def __init__(self, fn):
      self._fn = fn
      self._str = get_latex(fn)

    @property
    def __doc__(self):
      return self._fn.__doc__

    @__doc__.setter
    def __doc__(self, val):
      self._fn.__doc__ = val

    @property
    def __name__(self):
      return self._fn.__name__

    @__name__.setter
    def __name__(self, val):
      self._fn.__name__ = val

    def __call__(self, *args):
      return self._fn(*args)

    def __str__(self):
      return self._str

    def _repr_latex_(self):
      """
      Hooks into Jupyter notebook's display system.
      """
      return self._str

  return _LatexifiedFunction(fn)
<commit_msg>Adds braces.<commit_after># This is very scratchy and supports only limited portion of Python functions.

import ast
import math
import inspect

class LatexifyVisitor(ast.NodeVisitor):
  def generic_visit(self, node):
    return str(node)

  def visit_Module(self, node):
    return self.visit(node.body[0])

  def visit_FunctionDef(self, node):
    name_str = r'\mathrm{' + str(node.name) + '}'
    arg_strs = [str(arg.arg) for arg in node.args.args]
    body_str = self.visit(node.body[0])
    return name_str + '(' + ', '.join(arg_strs) + r') \triangleq ' + body_str

  def visit_Return(self, node):
    return self.visit(node.value)

  def visit_Call(self, node):
    builtin_callees = {
        'math.acos': (r'\arccos{\left({', r'}\right)}'),
        'math.acosh': (r'\operatorname{arccosh}{\left({', r'}\right)}'),
        'math.asin': (r'\arcsin{\left({', r'}\right)}'),
        'math.asinh': (r'\operatorname{arcsinh}{\left({', r'}\right)}'),
        'math.atan': (r'\arctan{\left({', r'}\right)}'),
        'math.atanh': (r'\operatorname{arctanh}{\left({', r'}\right)}'),
        'math.ceil': (r'\left\lceil{', r'}\right\rceil'),
        'math.cos': (r'\cos{\left({', r'}\right)}'),
        'math.cosh': (r'\cosh{\left({', r'}\right)}'),
        'math.exp': (r'\exp{\left({', r'}\right)'),
        'math.fabs': (r'\left|{', r'}\right|'),
        'math.factorial': (r'\left({', r'}\right)!'),
        'math.floor': (r'\left\lfloor{', r'}\right\rfloor'),
        'math.fsum': (r'\sum\left({', r'}\right)'),
        'math.gamma': (r'\Gamma\left({', r'}\right)'),
        'math.log': (r'\log{\left({', r'}\right)}'),
        'math.log10': (r'\log_{10}{\left({', r'}\right)'),
        'math.log2': (r'\log_{2}{\left({', r'}\right)'),
        'math.prod': (r'\prod \left({', r'}\right)'),
        'math.sin': (r'\sin{\left({', r'}\right)}'),
        'math.sinh': (r'\sinh{\left({', r'}\right)}'),
        'math.sqrt': (r'\sqrt{', '}'),
        'math.tan': (r'\tan{\left({', r'}\right)}'),
        'math.tanh': (r'\tanh{\left({', r'}\right)}'),
        'sum': (r'\sum \left({', r'}\right)'),
    }

    callee_str = self.visit(node.func)
    if callee_str in builtin_callees:
      lstr, rstr = builtin_callees[callee_str]
    else:
      if callee_str.startswith('math.'):
        callee_str = callee_str[5:]
      lstr = r'\operatorname{' + callee_str + '}\left('
      rstr = r'\right)'

    arg_strs = [self.visit(arg) for arg in node.args]
    return lstr + ', '.join(arg_strs) + rstr

  def visit_Attribute(self, node):
    vstr = self.visit(node.value)
    astr = str(node.attr)
    return vstr + '.' + astr

  def visit_Name(self, node):
    return str(node.id)

  def visit_Num(self, node):
    return str(node.n)

  def visit_UnaryOp(self, node):
    def _wrap(child):
      latex = self.visit(child)
      if isinstance(child, ast.BinOp) and isinstance(child.op, (ast.Add, ast.Sub)):
        return '(' + latex + ')'
      return latex

    reprs = {
        ast.UAdd: (lambda: _wrap(node.operand)),
        ast.USub: (lambda: '-' + _wrap(node.operand)),
    }

    if type(node.op) in reprs:
      return reprs[type(node.op)]()
    else:
      return r'\mathrm{unknown_uniop}(' + vstr + ')'

  def visit_BinOp(self, node):
    priority = {
        ast.Add: 10,
        ast.Sub: 10,
        ast.Mult: 20,
        ast.Div: 20,
        ast.Pow: 30,
    }

    def _unwrap(child):
      return self.visit(child)

    def _wrap(child):
      latex = _unwrap(child)
      if isinstance(child, ast.BinOp):
        cp = priority[type(child.op)] if type(child.op) in priority else 100
        pp = priority[type(node.op)] if type(node.op) in priority else 100
        if cp < pp:
          return '(' + latex + ')'
      return latex

    l = node.left
    r = node.right
    reprs = {
        ast.Add: (lambda: _wrap(l) + ' + ' + _wrap(r)),
        ast.Sub: (lambda: _wrap(l) + ' - ' + _wrap(r)),
        ast.Mult: (lambda: _wrap(l) + _wrap(r)),
        ast.Div: (lambda: r'\frac{' + _unwrap(l) + '}{' + _unwrap(r) + '}'),
        ast.Pow: (lambda: _wrap(l) + '^{' + _unwrap(r) + '}'),
    }

    if type(node.op) in reprs:
      return reprs[type(node.op)]()
    else:
      return r'\mathrm{unknown_binop}(' + lstr + ', ' + rstr + ')'

  def visit_Compare(self, node):
    lstr = self.visit(node.left)
    rstr = self.visit(node.comparators[0])

    if isinstance(node.ops[0], ast.Eq):
      return lstr + '=' + rstr
    else:
      return r'\mathrm{unknown_compop}(' + lstr + ', ' + rstr + ')'

  def visit_If(self, node):
    cond_str = self.visit(node.test)
    tstr = self.visit(node.body[0])
    fstr = self.visit(node.orelse[0])
    return r'\left\{ \begin{array}{ll} ' + tstr + r', & \mathrm{if} \ ' + cond_str + r' \\ ' + fstr + r', & \mathrm{otherwise} \end{array} \right.'

def get_latex(fn):
  return LatexifyVisitor().visit(ast.parse(inspect.getsource(fn)))

def with_latex(fn):

  class _LatexifiedFunction:
    def __init__(self, fn):
      self._fn = fn
      self._str = get_latex(fn)

    @property
    def __doc__(self):
      return self._fn.__doc__

    @__doc__.setter
    def __doc__(self, val):
      self._fn.__doc__ = val

    @property
    def __name__(self):
      return self._fn.__name__

    @__name__.setter
    def __name__(self, val):
      self._fn.__name__ = val

    def __call__(self, *args):
      return self._fn(*args)

    def __str__(self):
      return self._str

    def _repr_latex_(self):
      """
      Hooks into Jupyter notebook's display system.
      """
      return self._str

  return _LatexifiedFunction(fn)
<|endoftext|>


<commit_before>#!/usr/bin/env python

#This script is a wrapper for -fastq_mergepairs from USEARCH8
import os, sys, argparse, shutil, subprocess, glob, math, logging
import lib.fasta as fasta
import lib.fastq as fastq
import lib.primer as primer
import lib.revcomp_lib as revcomp_lib
import lib.progress as progress
import lib.die as die

class MyFormatter(argparse.ArgumentDefaultsHelpFormatter):
    def __init__(self,prog):
        super(MyFormatter,self).__init__(prog,max_help_position=48)
class col:
    GRN = '\033[92m'
    END = '\033[0m'
    WARN = '\033[93m'

parser=argparse.ArgumentParser(prog='ufits-process_illumina_folder.py', usage="%(prog)s [options] -i folder",
    description='''Script that takes De-mulitplexed Illumina data from a folder and processes it for ufits (merge PE reads, strip primers, trim/pad to set length.''',
    epilog="""Written by Jon Palmer (2015) palmer.jona@gmail.com""",
    formatter_class=MyFormatter)

parser.add_argument('-i','--input', dest='input', required=True, help='Folder of Illumina PE Data')
parser.add_argument('-o','--out', dest="out", default='ufits-results', help='Name for output folder')
parser.add_argument('-f','--fwd_primer', dest="F_primer", default='GTGARTCATCGAATCTTTG', help='Forward Primer (fITS7)')
parser.add_argument('-r','--rev_primer', dest="R_primer", default='TCCTCCGCTTATTGATATGC', help='Reverse Primer (ITS4)')
parser.add_argument('-n','--name_prefix', dest="prefix", default='R_', help='Prefix for renaming reads')
parser.add_argument('-m','--min_len', default='50', help='Minimum read length to keep')
parser.add_argument('-l','--trim_len', default='250', help='Trim length for reads')
parser.add_argument('-u','--usearch', dest="usearch", default='usearch8', help='USEARCH8 EXE')
args=parser.parse_args()

FNULL = open(os.devnull, 'w')

usearch = args.usearch
try:
    subprocess.call([usearch, '--version'], stdout = FNULL, stderr = FNULL)
except OSError:
    print "%s not found in your PATH, exiting." % usearch
    os._exit(1)

def convertSize(num, suffix='B'):
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

def myround(x, base=10):
    return int(base * round(float(x)/base))

def setupLogging(LOGNAME):
    global log
    stdoutformat = logging.Formatter(col.GRN+'%(asctime)s'+col.END+': %(message)s', datefmt='%b-%d-%Y %I:%M:%S %p')
    fileformat = logging.Formatter('%(asctime)s: %(message)s')
    log = logging.getLogger("myapp")
    log.setLevel(logging.DEBUG)
    sth = logging.StreamHandler()
    sth.setLevel(logging.INFO)
    sth.setFormatter(stdoutformat)
    log.addHandler(sth)
    fhnd = logging.FileHandler(LOGNAME)
    fhnd.setLevel(logging.DEBUG)
    fhnd.setFormatter(fileformat)
    log.addHandler(fhnd)

#create directory and check for existing logfile
if not os.path.exists(args.out):
    os.makedirs(args.out)

log_name = os.path.join(args.out, 'ufits.log')
if os.path.isfile(log_name):
    os.remove(log_name)

setupLogging(log_name)
cmd_args = " ".join(sys.argv)+'\n'
log.debug(cmd_args)
print "-------------------------------------------------------"

#get filenames, store in list
#Illumina file names look like the following
#<sample name>_<barcode sequence>_L<lane (0-padded to 3 digits)>_R<read number>_<set number (0-padded to 3 digits>.fastq.gz
filenames = []
for file in os.listdir(args.input):
    if file.endswith(".fastq"):
        filenames.append(file)

if len(filenames) % 2 != 0:
    print "Check your input files, they do not seem to be properly paired"
    os.exit(1)

uniq_names = []
fastq_for = []
fastq_rev = []
map = os.path.join(args.out, 'ufits-filenames.txt')
map_file = open(map, 'wb')
map_file.write("Name\t[i5]\t[i7]\tLane\tSet_num\n")
for item in sorted(filenames):
    if '_R1' in item:
        fastq_for.append(item)
    if '_R2' in item:
        fastq_rev.append(item)
    column = item.split("_")
    if column[0] not in uniq_names:
        uniq_names.append(column[0])
        if "-" in column[1]:
            barcode = column[1].split("-")
            try:
                map_file.write("%s\t%s\t%s\t%s\t%s\n" % (column[0], barcode[0], barcode[1], column[2], column[4].split(".",1)[0]))
            except IndexError:
                log.info("Non-standard names detected, skipping mapping file")
                break
        else:
            try:
                map_file.write("%s\t%s\t%s\t%s\t%s\n" % (column[0], column[1], "None", column[2], column[4].split(".",1)[0]))
            except IndexError:
                log.info("Non-standard names detected, skipping mapping file")
                break
map_file.close()

#loop through each set
for i in range(len(fastq_for)):
    name = fastq_for[i].split("_")[0]
    for_reads = os.path.join(args.input, fastq_for[i])
    rev_reads = os.path.join(args.input, fastq_rev[i])
    log.info("Working on reads from sample %s" % name)
    #get read length
    fp = open(for_reads)
    for i, line in enumerate(fp):
        if i == 1:
            read_length = len(line)
            read_length = myround(read_length)
        elif i > 2:
            break
    fp.close()

    #now trim the last bp off of the Illumina data (there for phasing, i.e. 250 bp reads are 251 bp)
    pretrim_R1 = os.path.join(args.out, 'pretrim_R1.fq')
    pretrim_R2 = os.path.join(args.out, 'pretrim_R2.fq')
    log.info("Merging Overlaping Pairs using USEARCH8")
    log.debug("%s -fastq_filter %s -fastq_trunclen %s -fastqout %s" % (usearch, for_reads, str(read_length), pretrim_R1))
    log.debug("%s -fastq_filter %s -fastq_trunclen %s -fastqout %s" % (usearch, rev_reads, str(read_length), pretrim_R2))
    subprocess.call([usearch, '-fastq_filter', for_reads, '-fastq_trunclen', str(read_length), '-fastqout', pretrim_R1], stdout = FNULL, stderr = FNULL)
    subprocess.call([usearch, '-fastq_filter', rev_reads, '-fastq_trunclen', str(read_length), '-fastqout', pretrim_R2], stdout = FNULL, stderr = FNULL)

    #next run USEARCH8 mergepe
    merge_out = os.path.join(args.out, 'merged.fq')
    skip_for = os.path.join(args.out, 'notmerged.R1.fq')
    log.debug("%s -fastq_mergepairs %s -reverse %s -fastqout %s -fastqout_notmerged_fwd %s -fastq_truncqual 5 -fastq_allowmergestagger -minhsp 12" % (usearch, pretrim_R1, pretrim_R2, merge_out, skip_for))
    subprocess.call([usearch, '-fastq_mergepairs', for_reads, '-reverse', rev_reads, '-fastqout', merge_out, '-fastqout_notmerged_fwd', skip_for, '-fastq_truncqual', '5','-fastq_allowmergestagger','-minhsp', '12'], stdout = FNULL, stderr = FNULL)

    #now concatenate files for downstream pre-process_illumina.py script
    outname = name + '.fq'
    final_out = os.path.join(args.out, outname)
    out_file = open(final_out, 'wb')
    shutil.copyfileobj(open(merge_out,'rb'), out_file)
    shutil.copyfileobj(open(skip_for,'rb'), out_file)
    out_file.close()

    #clean and close up intermediate files
    os.remove(merge_out)
    os.remove(pretrim_R1)
    os.remove(pretrim_R2)
    os.remove(skip_for)

    log.info("Strip primers, trim/pad to %s bp\n" % args.trim_len)

    #now rest of script for demultiplexing here
    MAX_PRIMER_MISMATCHES = 2
    FileName = final_out
    FwdPrimer = args.F_primer
    RevPrimer = args.R_primer
    LabelPrefix = args.prefix
    SampleLabel = name
    MinLen = int(args.min_len)
    TrimLen = int(args.trim_len)
    demuxname = name + '.demux.fq'
    DemuxOut = os.path.join(args.out, demuxname)
    out_file = open(DemuxOut, 'w')

    SeqCount = 0
    OutCount = 0
    FwdPrimerMismatchCount = 0
    RevPrimerStrippedCount = 0
    TooShortCount = 0
    PadCount = 0

    PL = len(FwdPrimer)
    RevPrimer = revcomp_lib.RevComp(RevPrimer)

    print >> sys.stderr, "Foward primer: ", FwdPrimer
    print >> sys.stderr, "Rev comp'd rev primer: ", RevPrimer

    def MatchesPrimer(Seq, Primer):
        return primer.MatchPrefix(Seq, Primer)

    def OnRec(Label, Seq, Qual):
        global PL, LabelPrefix, SeqCount, OutCount, TooShortCount, PadCount
        global FwdPrimerMismatchCount, RevPrimerStrippedCount
        global FwdPrimer, RevPrimer

        if SeqCount == 0:
            progress.InitFile(fastq.File)

        progress.File("%u reads, %u outupt, %u bad fwd primer, %u rev primer stripped, %u too short. %u padded" % \
          (SeqCount, OutCount, FwdPrimerMismatchCount, RevPrimerStrippedCount, TooShortCount, PadCount))

        SeqCount += 1
        Seq = Seq
        Qual = Qual
        Diffs = MatchesPrimer(Seq, FwdPrimer)
        if Diffs > MAX_PRIMER_MISMATCHES:
            FwdPrimerMismatchCount += 1
            return

        OutCount += 1
        Label = LabelPrefix + str(OutCount) + ";barcodelabel=" + SampleLabel + ";"

    # Strip fwd primer
        Seq = Seq[PL:]
        Qual = Qual[PL:]

        BestPosRev, BestDiffsRev = primer.BestMatch2(Seq, RevPrimer, MAX_PRIMER_MISMATCHES)
        if BestPosRev > 0:
            # Strip rev primer
            RevPrimerStrippedCount += 1
            StrippedSeq = Seq[:BestPosRev]
            StrippedQual = Qual[:BestPosRev]

            # correctness checks
            if 1:
                Tail = Seq[BestPosRev:]
                Diffs2 = primer.MatchPrefix(Tail, RevPrimer)
                if Diffs2 != BestDiffsRev:
                    print >> sys.stderr
                    print >> sys.stderr, " Seq=" + Seq
                    print >> sys.stderr, "Tail=" + Tail
                    print >> sys.stderr, "RevP=" + RevPrimer
                    die.Die("BestPosRev %u Diffs2 %u BestDiffsRev %u" % (BestPosRev, Diffs2, BestDiffsRev))
                assert StrippedSeq + Tail == Seq

            Seq = StrippedSeq
            Qual = StrippedQual

            L = len(Seq)
            assert len(Qual) == L

            if L < MinLen:
                return

            if L < TrimLen:
                PadCount += 1
                Seq = Seq + (TrimLen - L)*'N'
                Qual = Qual + (TrimLen - L)*'I'
                L = len(Seq)
                assert L == TrimLen
                assert len(Qual) == TrimLen

        L = len(Seq)
        if L < TrimLen:
            TooShortCount += 1
            return

        if L > TrimLen:
            Seq = Seq[:TrimLen]
            Qual = Qual[:TrimLen]
            L = len(Seq)

        assert L == TrimLen
        assert len(Qual) == TrimLen

        fastq.WriteRec(out_file, Label, Seq, Qual)

    fastq.ReadRecs(FileName, OnRec)
    progress.FileDone("%u reads, %u outupt, %u bad fwd primer, %u rev primer stripped, %u too short" % \
          (SeqCount, OutCount, FwdPrimerMismatchCount, RevPrimerStrippedCount, TooShortCount))

    print >> sys.stderr, "%10u seqs" % SeqCount
    print >> sys.stderr, "%10u fwd primer mismatches (%.1f%% discarded)" % (FwdPrimerMismatchCount, FwdPrimerMismatchCount*100.0/SeqCount)
    print >> sys.stderr, "%10u rev primer stripped (%.2f%% kept)" % (RevPrimerStrippedCount, RevPrimerStrippedCount*100.0/SeqCount)
    print >> sys.stderr, "%10u padded (%.2f%%)" % (PadCount, PadCount*100.0/SeqCount)
    print >> sys.stderr, "%10u too short (%.2f%%)" % (TooShortCount, TooShortCount*100.0/SeqCount)
    print >> sys.stderr, "%10u output (%.1f%%)\n" % (OutCount, OutCount*100.0/SeqCount)
    out_file.close()

print "-------------------------------------------------------"
#Now concatenate all of the demuxed files together
log.info("Concatenating Demuxed Files")

catDemux = os.path.join(args.out, 'ufits.demux.fq')
with open(catDemux, 'wb') as outfile:
    for filename in glob.glob(os.path.join(args.out,'*.demux.fq')):
        if filename == catDemux:
            continue
        with open(filename, 'rb') as readfile:
            shutil.copyfileobj(readfile, outfile)

log.info("Counting FASTQ Records")
num_lines = sum(1 for line in open(catDemux))
total = int(num_lines) / 4
log.info('{0:,}'.format(total) + ' reads processed')
log.info("Output file:  %s" % catDemux)

#get file size and issue warning if over 4.0 GB
filesize = os.path.getsize(catDemux)
readablesize = convertSize(filesize)
log.info("File size:  %s" % readablesize)
if filesize >= 4294967296:
    print col.WARN + "Warning, file is larger than 4 GB, you will need USEARCH 64 bit to cluster OTUs" + col.END

    <commit_msg>bug fix<commit_after>#!/usr/bin/env python

#This script is a wrapper for -fastq_mergepairs from USEARCH8
import os, sys, argparse, shutil, subprocess, glob, math, logging
import lib.fasta as fasta
import lib.fastq as fastq
import lib.primer as primer
import lib.revcomp_lib as revcomp_lib
import lib.progress as progress
import lib.die as die

class MyFormatter(argparse.ArgumentDefaultsHelpFormatter):
    def __init__(self,prog):
        super(MyFormatter,self).__init__(prog,max_help_position=48)
class col:
    GRN = '\033[92m'
    END = '\033[0m'
    WARN = '\033[93m'

parser=argparse.ArgumentParser(prog='ufits-process_illumina_folder.py', usage="%(prog)s [options] -i folder",
    description='''Script that takes De-mulitplexed Illumina data from a folder and processes it for ufits (merge PE reads, strip primers, trim/pad to set length.''',
    epilog="""Written by Jon Palmer (2015) palmer.jona@gmail.com""",
    formatter_class=MyFormatter)

parser.add_argument('-i','--input', dest='input', required=True, help='Folder of Illumina PE Data')
parser.add_argument('-o','--out', dest="out", default='ufits-results', help='Name for output folder')
parser.add_argument('-f','--fwd_primer', dest="F_primer", default='GTGARTCATCGAATCTTTG', help='Forward Primer (fITS7)')
parser.add_argument('-r','--rev_primer', dest="R_primer", default='TCCTCCGCTTATTGATATGC', help='Reverse Primer (ITS4)')
parser.add_argument('-n','--name_prefix', dest="prefix", default='R_', help='Prefix for renaming reads')
parser.add_argument('-m','--min_len', default='50', help='Minimum read length to keep')
parser.add_argument('-l','--trim_len', default='250', help='Trim length for reads')
parser.add_argument('-u','--usearch', dest="usearch", default='usearch8', help='USEARCH8 EXE')
args=parser.parse_args()

FNULL = open(os.devnull, 'w')

usearch = args.usearch
try:
    subprocess.call([usearch, '--version'], stdout = FNULL, stderr = FNULL)
except OSError:
    print "%s not found in your PATH, exiting." % usearch
    os._exit(1)

def convertSize(num, suffix='B'):
    for unit in ['','K','M','G','T','P','E','Z']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f%s%s" % (num, 'Y', suffix)

def myround(x, base=10):
    return int(base * round(float(x)/base))

def setupLogging(LOGNAME):
    global log
    stdoutformat = logging.Formatter(col.GRN+'%(asctime)s'+col.END+': %(message)s', datefmt='%b-%d-%Y %I:%M:%S %p')
    fileformat = logging.Formatter('%(asctime)s: %(message)s')
    log = logging.getLogger("myapp")
    log.setLevel(logging.DEBUG)
    sth = logging.StreamHandler()
    sth.setLevel(logging.INFO)
    sth.setFormatter(stdoutformat)
    log.addHandler(sth)
    fhnd = logging.FileHandler(LOGNAME)
    fhnd.setLevel(logging.DEBUG)
    fhnd.setFormatter(fileformat)
    log.addHandler(fhnd)

#create directory and check for existing logfile
if not os.path.exists(args.out):
    os.makedirs(args.out)

log_name = os.path.join(args.out, 'ufits.log')
if os.path.isfile(log_name):
    os.remove(log_name)

setupLogging(log_name)
cmd_args = " ".join(sys.argv)+'\n'
log.debug(cmd_args)
print "-------------------------------------------------------"

#get filenames, store in list
#Illumina file names look like the following
#<sample name>_<barcode sequence>_L<lane (0-padded to 3 digits)>_R<read number>_<set number (0-padded to 3 digits>.fastq.gz
filenames = []
for file in os.listdir(args.input):
    if file.endswith(".fastq"):
        filenames.append(file)

if len(filenames) % 2 != 0:
    print "Check your input files, they do not seem to be properly paired"
    os.exit(1)

uniq_names = []
fastq_for = []
fastq_rev = []
map = os.path.join(args.out, 'ufits-filenames.txt')
map_file = open(map, 'wb')
map_file.write("Name\t[i5]\t[i7]\tLane\tSet_num\n")
for item in sorted(filenames):
    if '_R1' in item:
        fastq_for.append(item)
    if '_R2' in item:
        fastq_rev.append(item)
    column = item.split("_")
    if column[0] not in uniq_names:
        uniq_names.append(column[0])
        if "-" in column[1]:
            barcode = column[1].split("-")
            try:
                map_file.write("%s\t%s\t%s\t%s\t%s\n" % (column[0], barcode[0], barcode[1], column[2], column[4].split(".",1)[0]))
            except IndexError:
                break
        else:
            try:
                map_file.write("%s\t%s\t%s\t%s\t%s\n" % (column[0], column[1], "None", column[2], column[4].split(".",1)[0]))
            except IndexError:
                break
map_file.close()

#loop through each set
for i in range(len(fastq_for)):
    name = fastq_for[i].split("_")[0]
    for_reads = os.path.join(args.input, fastq_for[i])
    rev_reads = os.path.join(args.input, fastq_rev[i])
    log.info("Working on reads from sample %s" % name)
    #get read length
    fp = open(for_reads)
    for i, line in enumerate(fp):
        if i == 1:
            read_length = len(line)
            read_length = myround(read_length)
        elif i > 2:
            break
    fp.close()

    #now trim the last bp off of the Illumina data (there for phasing, i.e. 250 bp reads are 251 bp)
    pretrim_R1 = os.path.join(args.out, 'pretrim_R1.fq')
    pretrim_R2 = os.path.join(args.out, 'pretrim_R2.fq')
    log.info("Merging Overlaping Pairs using USEARCH8")
    log.debug("%s -fastq_filter %s -fastq_trunclen %s -fastqout %s" % (usearch, for_reads, str(read_length), pretrim_R1))
    log.debug("%s -fastq_filter %s -fastq_trunclen %s -fastqout %s" % (usearch, rev_reads, str(read_length), pretrim_R2))
    subprocess.call([usearch, '-fastq_filter', for_reads, '-fastq_trunclen', str(read_length), '-fastqout', pretrim_R1], stdout = FNULL, stderr = FNULL)
    subprocess.call([usearch, '-fastq_filter', rev_reads, '-fastq_trunclen', str(read_length), '-fastqout', pretrim_R2], stdout = FNULL, stderr = FNULL)

    #next run USEARCH8 mergepe
    merge_out = os.path.join(args.out, 'merged.fq')
    skip_for = os.path.join(args.out, 'notmerged.R1.fq')
    log.debug("%s -fastq_mergepairs %s -reverse %s -fastqout %s -fastqout_notmerged_fwd %s -fastq_truncqual 5 -fastq_allowmergestagger -minhsp 12" % (usearch, pretrim_R1, pretrim_R2, merge_out, skip_for))
    subprocess.call([usearch, '-fastq_mergepairs', for_reads, '-reverse', rev_reads, '-fastqout', merge_out, '-fastqout_notmerged_fwd', skip_for, '-fastq_truncqual', '5','-fastq_allowmergestagger','-minhsp', '12'], stdout = FNULL, stderr = FNULL)

    #now concatenate files for downstream pre-process_illumina.py script
    outname = name + '.fq'
    final_out = os.path.join(args.out, outname)
    out_file = open(final_out, 'wb')
    shutil.copyfileobj(open(merge_out,'rb'), out_file)
    shutil.copyfileobj(open(skip_for,'rb'), out_file)
    out_file.close()

    #clean and close up intermediate files
    os.remove(merge_out)
    os.remove(pretrim_R1)
    os.remove(pretrim_R2)
    os.remove(skip_for)

    log.info("Strip primers, trim/pad to %s bp\n" % args.trim_len)

    #now rest of script for demultiplexing here
    MAX_PRIMER_MISMATCHES = 2
    FileName = final_out
    FwdPrimer = args.F_primer
    RevPrimer = args.R_primer
    LabelPrefix = args.prefix
    SampleLabel = name
    MinLen = int(args.min_len)
    TrimLen = int(args.trim_len)
    demuxname = name + '.demux.fq'
    DemuxOut = os.path.join(args.out, demuxname)
    out_file = open(DemuxOut, 'w')

    SeqCount = 0
    OutCount = 0
    FwdPrimerMismatchCount = 0
    RevPrimerStrippedCount = 0
    TooShortCount = 0
    PadCount = 0

    PL = len(FwdPrimer)
    RevPrimer = revcomp_lib.RevComp(RevPrimer)

    print >> sys.stderr, "Foward primer: ", FwdPrimer
    print >> sys.stderr, "Rev comp'd rev primer: ", RevPrimer

    def MatchesPrimer(Seq, Primer):
        return primer.MatchPrefix(Seq, Primer)

    def OnRec(Label, Seq, Qual):
        global PL, LabelPrefix, SeqCount, OutCount, TooShortCount, PadCount
        global FwdPrimerMismatchCount, RevPrimerStrippedCount
        global FwdPrimer, RevPrimer

        if SeqCount == 0:
            progress.InitFile(fastq.File)

        progress.File("%u reads, %u outupt, %u bad fwd primer, %u rev primer stripped, %u too short. %u padded" % \
          (SeqCount, OutCount, FwdPrimerMismatchCount, RevPrimerStrippedCount, TooShortCount, PadCount))

        SeqCount += 1
        Seq = Seq
        Qual = Qual
        Diffs = MatchesPrimer(Seq, FwdPrimer)
        if Diffs > MAX_PRIMER_MISMATCHES:
            FwdPrimerMismatchCount += 1
            return

        OutCount += 1
        Label = LabelPrefix + str(OutCount) + ";barcodelabel=" + SampleLabel + ";"

    # Strip fwd primer
        Seq = Seq[PL:]
        Qual = Qual[PL:]

        BestPosRev, BestDiffsRev = primer.BestMatch2(Seq, RevPrimer, MAX_PRIMER_MISMATCHES)
        if BestPosRev > 0:
            # Strip rev primer
            RevPrimerStrippedCount += 1
            StrippedSeq = Seq[:BestPosRev]
            StrippedQual = Qual[:BestPosRev]

            # correctness checks
            if 1:
                Tail = Seq[BestPosRev:]
                Diffs2 = primer.MatchPrefix(Tail, RevPrimer)
                if Diffs2 != BestDiffsRev:
                    print >> sys.stderr
                    print >> sys.stderr, " Seq=" + Seq
                    print >> sys.stderr, "Tail=" + Tail
                    print >> sys.stderr, "RevP=" + RevPrimer
                    die.Die("BestPosRev %u Diffs2 %u BestDiffsRev %u" % (BestPosRev, Diffs2, BestDiffsRev))
                assert StrippedSeq + Tail == Seq

            Seq = StrippedSeq
            Qual = StrippedQual

            L = len(Seq)
            assert len(Qual) == L

            if L < MinLen:
                return

            if L < TrimLen:
                PadCount += 1
                Seq = Seq + (TrimLen - L)*'N'
                Qual = Qual + (TrimLen - L)*'I'
                L = len(Seq)
                assert L == TrimLen
                assert len(Qual) == TrimLen

        L = len(Seq)
        if L < TrimLen:
            TooShortCount += 1
            return

        if L > TrimLen:
            Seq = Seq[:TrimLen]
            Qual = Qual[:TrimLen]
            L = len(Seq)

        assert L == TrimLen
        assert len(Qual) == TrimLen

        fastq.WriteRec(out_file, Label, Seq, Qual)

    fastq.ReadRecs(FileName, OnRec)
    progress.FileDone("%u reads, %u outupt, %u bad fwd primer, %u rev primer stripped, %u too short" % \
          (SeqCount, OutCount, FwdPrimerMismatchCount, RevPrimerStrippedCount, TooShortCount))

    print >> sys.stderr, "%10u seqs" % SeqCount
    print >> sys.stderr, "%10u fwd primer mismatches (%.1f%% discarded)" % (FwdPrimerMismatchCount, FwdPrimerMismatchCount*100.0/SeqCount)
    print >> sys.stderr, "%10u rev primer stripped (%.2f%% kept)" % (RevPrimerStrippedCount, RevPrimerStrippedCount*100.0/SeqCount)
    print >> sys.stderr, "%10u padded (%.2f%%)" % (PadCount, PadCount*100.0/SeqCount)
    print >> sys.stderr, "%10u too short (%.2f%%)" % (TooShortCount, TooShortCount*100.0/SeqCount)
    print >> sys.stderr, "%10u output (%.1f%%)\n" % (OutCount, OutCount*100.0/SeqCount)
    out_file.close()

print "-------------------------------------------------------"
#Now concatenate all of the demuxed files together
log.info("Concatenating Demuxed Files")

catDemux = os.path.join(args.out, 'ufits.demux.fq')
with open(catDemux, 'wb') as outfile:
    for filename in glob.glob(os.path.join(args.out,'*.demux.fq')):
        if filename == catDemux:
            continue
        with open(filename, 'rb') as readfile:
            shutil.copyfileobj(readfile, outfile)

log.info("Counting FASTQ Records")
num_lines = sum(1 for line in open(catDemux))
total = int(num_lines) / 4
log.info('{0:,}'.format(total) + ' reads processed')
log.info("Output file:  %s" % catDemux)

#get file size and issue warning if over 4.0 GB
filesize = os.path.getsize(catDemux)
readablesize = convertSize(filesize)
log.info("File size:  %s" % readablesize)
if filesize >= 4294967296:
    print col.WARN + "Warning, file is larger than 4 GB, you will need USEARCH 64 bit to cluster OTUs" + col.END

    <|endoftext|>


<commit_before># This file is part of h5py, a Python interface to the HDF5 library.
#
# http://www.h5py.org
#
# Copyright 2008-2013 Andrew Collette and contributors
#
# License:  Standard 3-clause BSD; see "license.txt" for full license terms
#           and contributor agreement.

"""
    Attribute data transfer testing module

    Covers all data read/write and type-conversion operations for attributes.
"""

from __future__ import absolute_import

import six

import numpy as np

from ..common import TestCase, ut

import h5py
from h5py import h5a, h5s, h5t
from h5py import File
from h5py._hl.base import is_empty_dataspace

class BaseAttrs(TestCase):

    def setUp(self):
        self.f = File(self.mktemp(), 'w')

    def tearDown(self):
        if self.f:
            self.f.close()

class TestScalar(BaseAttrs):

    """
        Feature: Scalar types map correctly to array scalars
    """

    def test_int(self):
        """ Integers are read as correct NumPy type """
        self.f.attrs['x'] = np.array(1, dtype=np.int8)
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.int8)

    def test_compound(self):
        """ Compound scalars are read as numpy.void """
        dt = np.dtype([('a','i'),('b','f')])
        data = np.array((1,4.2), dtype=dt)
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.void)
        self.assertEqual(out, data)
        self.assertEqual(out['b'], data['b'])

class TestArray(BaseAttrs):

    """
        Feature: Non-scalar types are correctly retrieved as ndarrays
    """

    def test_single(self):
        """ Single-element arrays are correctly recovered """
        data = np.ndarray((1,), dtype='f')
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.ndarray)
        self.assertEqual(out.shape, (1,))

    def test_multi(self):
        """ Rank-1 arrays are correctly recovered """
        data = np.ndarray((42,), dtype='f')
        data[:] = 42.0
        data[10:35] = -47.0
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.ndarray)
        self.assertEqual(out.shape, (42,))
        self.assertArrayEqual(out, data)

class TestTypes(BaseAttrs):

    """
        Feature: All supported types can be stored in attributes
    """

    def test_int(self):
        """ Storage of integer types """
        dtypes = (np.int8, np.int16, np.int32, np.int64,
                  np.uint8, np.uint16, np.uint32, np.uint64)
        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 42
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_float(self):
        """ Storage of floating point types """
        dtypes = tuple(np.dtype(x) for x in ('<f4','>f4','>f8','<f8'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 42.3
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            #TODO: Clean up after issue addressed !
            print("dtype: ", out.dtype, dt)
            print("value: ", out, data)
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_complex(self):
        """ Storage of complex types """
        dtypes = tuple(np.dtype(x) for x in ('<c8','>c8','<c16','>c16'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = -4.2j+35.9
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            print("dtype: ", out.dtype, dt)
            print("value: ", out, data)
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_string(self):
        """ Storage of fixed-length strings """
        dtypes = tuple(np.dtype(x) for x in ('|S1', '|S10'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 'h'
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            self.assertEqual(out.dtype, dt)
            self.assertEqual(out[0], data[0])

    def test_bool(self):
        """ Storage of NumPy booleans """

        data = np.ndarray((2,), dtype=np.bool_)
        data[...] = True, False
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertEqual(out.dtype, data.dtype)
        self.assertEqual(out[0], data[0])
        self.assertEqual(out[1], data[1])

    def test_vlen_string_array(self):
        """ Storage of vlen byte string arrays"""
        dt = h5py.string_dtype(encoding='ascii')

        data = np.ndarray((2,), dtype=dt)
        data[...] = b"Hello", b"Hi there!  This is HDF5!"

        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertEqual(out.dtype, dt)
        self.assertEqual(out[0], data[0])
        self.assertEqual(out[1], data[1])

    def test_string_scalar(self):
        """ Storage of variable-length byte string scalars (auto-creation) """

        self.f.attrs['x'] = b'Hello'
        out = self.f.attrs['x']

        self.assertEqual(out,b'Hello')
        self.assertEqual(type(out), bytes)

        aid = h5py.h5a.open(self.f.id, b"x")
        tid = aid.get_type()
        self.assertEqual(type(tid), h5py.h5t.TypeStringID)
        self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII)
        self.assertTrue(tid.is_variable_str())

    def test_unicode_scalar(self):
        """ Storage of variable-length unicode strings (auto-creation) """

        self.f.attrs['x'] = u"Hello" + six.unichr(0x2340) + u"!!"
        out = self.f.attrs['x']
        self.assertEqual(out, u"Hello" + six.unichr(0x2340) + u"!!")
        self.assertEqual(type(out), six.text_type)

        aid = h5py.h5a.open(self.f.id, b"x")
        tid = aid.get_type()
        self.assertEqual(type(tid), h5py.h5t.TypeStringID)
        self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8)
        self.assertTrue(tid.is_variable_str())

class TestEmpty(BaseAttrs):

    def setUp(self):
        BaseAttrs.setUp(self)
        sid = h5s.create(h5s.NULL)
        tid = h5t.C_S1.copy()
        tid.set_size(10)
        aid = h5a.create(self.f.id, b'x', tid, sid)
        self.empty_obj = h5py.Empty(np.dtype("S10"))

    def test_read(self):
        self.assertEqual(
            self.empty_obj, self.f.attrs['x']
        )

    def test_write(self):
        self.f.attrs["y"] = self.empty_obj
        self.assertTrue(is_empty_dataspace(h5a.open(self.f.id, b'y')))

    def test_modify(self):
        with self.assertRaises(IOError):
            self.f.attrs.modify('x', 1)

    def test_values(self):
        # list() is for Py3 where these are iterators
        values = list(self.f.attrs.values())
        self.assertEqual(
            [self.empty_obj], values
        )

    def test_items(self):
        items = list(self.f.attrs.items())
        self.assertEqual(
            [(u"x", self.empty_obj)], items
        )

    def test_itervalues(self):
        values = list(six.itervalues(self.f.attrs))
        self.assertEqual(
            [self.empty_obj], values
        )

    def test_iteritems(self):
        items = list(six.iteritems(self.f.attrs))
        self.assertEqual(
            [(u"x", self.empty_obj)], items
        )

class TestWriteException(BaseAttrs):

    """
        Ensure failed attribute writes don't leave garbage behind.
    """

    def test_write(self):
        """ ValueError on string write wipes out attribute """

        s = b"Hello\x00Hello"

        try:
            self.f.attrs['x'] = s
        except ValueError:
            pass

        with self.assertRaises(KeyError):
            self.f.attrs['x']
<commit_msg>cleanup un-needed debug statement<commit_after># This file is part of h5py, a Python interface to the HDF5 library.
#
# http://www.h5py.org
#
# Copyright 2008-2013 Andrew Collette and contributors
#
# License:  Standard 3-clause BSD; see "license.txt" for full license terms
#           and contributor agreement.

"""
    Attribute data transfer testing module

    Covers all data read/write and type-conversion operations for attributes.
"""

from __future__ import absolute_import

import six

import numpy as np

from ..common import TestCase, ut

import h5py
from h5py import h5a, h5s, h5t
from h5py import File
from h5py._hl.base import is_empty_dataspace

class BaseAttrs(TestCase):

    def setUp(self):
        self.f = File(self.mktemp(), 'w')

    def tearDown(self):
        if self.f:
            self.f.close()

class TestScalar(BaseAttrs):

    """
        Feature: Scalar types map correctly to array scalars
    """

    def test_int(self):
        """ Integers are read as correct NumPy type """
        self.f.attrs['x'] = np.array(1, dtype=np.int8)
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.int8)

    def test_compound(self):
        """ Compound scalars are read as numpy.void """
        dt = np.dtype([('a','i'),('b','f')])
        data = np.array((1,4.2), dtype=dt)
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.void)
        self.assertEqual(out, data)
        self.assertEqual(out['b'], data['b'])

class TestArray(BaseAttrs):

    """
        Feature: Non-scalar types are correctly retrieved as ndarrays
    """

    def test_single(self):
        """ Single-element arrays are correctly recovered """
        data = np.ndarray((1,), dtype='f')
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.ndarray)
        self.assertEqual(out.shape, (1,))

    def test_multi(self):
        """ Rank-1 arrays are correctly recovered """
        data = np.ndarray((42,), dtype='f')
        data[:] = 42.0
        data[10:35] = -47.0
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertIsInstance(out, np.ndarray)
        self.assertEqual(out.shape, (42,))
        self.assertArrayEqual(out, data)

class TestTypes(BaseAttrs):

    """
        Feature: All supported types can be stored in attributes
    """

    def test_int(self):
        """ Storage of integer types """
        dtypes = (np.int8, np.int16, np.int32, np.int64,
                  np.uint8, np.uint16, np.uint32, np.uint64)
        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 42
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_float(self):
        """ Storage of floating point types """
        dtypes = tuple(np.dtype(x) for x in ('<f4','>f4','>f8','<f8'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 42.3
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            #TODO: Clean up after issue addressed !
            print("dtype: ", out.dtype, dt)
            print("value: ", out, data)
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_complex(self):
        """ Storage of complex types """
        dtypes = tuple(np.dtype(x) for x in ('<c8','>c8','<c16','>c16'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = -4.2j+35.9
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            self.assertEqual(out.dtype, dt)
            self.assertArrayEqual(out, data)

    def test_string(self):
        """ Storage of fixed-length strings """
        dtypes = tuple(np.dtype(x) for x in ('|S1', '|S10'))

        for dt in dtypes:
            data = np.ndarray((1,), dtype=dt)
            data[...] = 'h'
            self.f.attrs['x'] = data
            out = self.f.attrs['x']
            self.assertEqual(out.dtype, dt)
            self.assertEqual(out[0], data[0])

    def test_bool(self):
        """ Storage of NumPy booleans """

        data = np.ndarray((2,), dtype=np.bool_)
        data[...] = True, False
        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertEqual(out.dtype, data.dtype)
        self.assertEqual(out[0], data[0])
        self.assertEqual(out[1], data[1])

    def test_vlen_string_array(self):
        """ Storage of vlen byte string arrays"""
        dt = h5py.string_dtype(encoding='ascii')

        data = np.ndarray((2,), dtype=dt)
        data[...] = b"Hello", b"Hi there!  This is HDF5!"

        self.f.attrs['x'] = data
        out = self.f.attrs['x']
        self.assertEqual(out.dtype, dt)
        self.assertEqual(out[0], data[0])
        self.assertEqual(out[1], data[1])

    def test_string_scalar(self):
        """ Storage of variable-length byte string scalars (auto-creation) """

        self.f.attrs['x'] = b'Hello'
        out = self.f.attrs['x']

        self.assertEqual(out,b'Hello')
        self.assertEqual(type(out), bytes)

        aid = h5py.h5a.open(self.f.id, b"x")
        tid = aid.get_type()
        self.assertEqual(type(tid), h5py.h5t.TypeStringID)
        self.assertEqual(tid.get_cset(), h5py.h5t.CSET_ASCII)
        self.assertTrue(tid.is_variable_str())

    def test_unicode_scalar(self):
        """ Storage of variable-length unicode strings (auto-creation) """

        self.f.attrs['x'] = u"Hello" + six.unichr(0x2340) + u"!!"
        out = self.f.attrs['x']
        self.assertEqual(out, u"Hello" + six.unichr(0x2340) + u"!!")
        self.assertEqual(type(out), six.text_type)

        aid = h5py.h5a.open(self.f.id, b"x")
        tid = aid.get_type()
        self.assertEqual(type(tid), h5py.h5t.TypeStringID)
        self.assertEqual(tid.get_cset(), h5py.h5t.CSET_UTF8)
        self.assertTrue(tid.is_variable_str())

class TestEmpty(BaseAttrs):

    def setUp(self):
        BaseAttrs.setUp(self)
        sid = h5s.create(h5s.NULL)
        tid = h5t.C_S1.copy()
        tid.set_size(10)
        aid = h5a.create(self.f.id, b'x', tid, sid)
        self.empty_obj = h5py.Empty(np.dtype("S10"))

    def test_read(self):
        self.assertEqual(
            self.empty_obj, self.f.attrs['x']
        )

    def test_write(self):
        self.f.attrs["y"] = self.empty_obj
        self.assertTrue(is_empty_dataspace(h5a.open(self.f.id, b'y')))

    def test_modify(self):
        with self.assertRaises(IOError):
            self.f.attrs.modify('x', 1)

    def test_values(self):
        # list() is for Py3 where these are iterators
        values = list(self.f.attrs.values())
        self.assertEqual(
            [self.empty_obj], values
        )

    def test_items(self):
        items = list(self.f.attrs.items())
        self.assertEqual(
            [(u"x", self.empty_obj)], items
        )

    def test_itervalues(self):
        values = list(six.itervalues(self.f.attrs))
        self.assertEqual(
            [self.empty_obj], values
        )

    def test_iteritems(self):
        items = list(six.iteritems(self.f.attrs))
        self.assertEqual(
            [(u"x", self.empty_obj)], items
        )

class TestWriteException(BaseAttrs):

    """
        Ensure failed attribute writes don't leave garbage behind.
    """

    def test_write(self):
        """ ValueError on string write wipes out attribute """

        s = b"Hello\x00Hello"

        try:
            self.f.attrs['x'] = s
        except ValueError:
            pass

        with self.assertRaises(KeyError):
            self.f.attrs['x']
<|endoftext|>


<commit_before>import os

from flask import Flask, request, Response
from flask import render_template, url_for, redirect, send_from_directory
from flask import send_file, make_response, abort

from angular_flask import app

# routing for API endpoints, generated from the models designated as API_MODELS
from angular_flask.core import api_manager
from angular_flask.models import *

for model_name in app.config['API_MODELS']:
    model_class = app.config['API_MODELS'][model_name]
    api_manager.create_api(model_class, methods=['GET', 'POST'])

session = api_manager.session

# routing for basic pages (pass routing onto the Angular app)
@app.route('/')
@app.route('/about')
@app.route('/blog')
def basic_pages(**kwargs):
    return make_response(open('angular_flask/templates/index.html').read())

# routing for CRUD-style endpoints
# passes routing onto the angular frontend if the requested resource exists
from sqlalchemy.sql import exists

crud_url_models = app.config['CRUD_URL_MODELS']

@app.route('/<model_name>/')
@app.route('/<model_name>/<item_id>')
def rest_pages(model_name, item_id=None):
    if model_name in crud_url_models:
        model_class = crud_url_models[model_name]
        if item_id is None or session.query(exists().where(
                model_class.id == item_id)).scalar():
            return make_response(open(
                'angular_flask/templates/index.html').read())
    abort(404)

# special file handlers and error handlers
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'img/favicon.ico')

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404
<commit_msg>Added songs route<commit_after>import os

from flask import Flask, request, Response
from flask import render_template, url_for, redirect, send_from_directory
from flask import send_file, make_response, abort

from angular_flask import app

# routing for API endpoints, generated from the models designated as API_MODELS
from angular_flask.core import api_manager
from angular_flask.models import *

for model_name in app.config['API_MODELS']:
    model_class = app.config['API_MODELS'][model_name]
    api_manager.create_api(model_class, methods=['GET', 'POST'])

session = api_manager.session

# routing for basic pages (pass routing onto the Angular app)
@app.route('/')
@app.route('/about')
@app.route('/blog')
@app.route('/songs')
def basic_pages(**kwargs):
    return make_response(open('angular_flask/templates/index.html').read())

# routing for CRUD-style endpoints
# passes routing onto the angular frontend if the requested resource exists
from sqlalchemy.sql import exists

crud_url_models = app.config['CRUD_URL_MODELS']

@app.route('/<model_name>/')
@app.route('/<model_name>/<item_id>')
def rest_pages(model_name, item_id=None):
    if model_name in crud_url_models:
        model_class = crud_url_models[model_name]
        if item_id is None or session.query(exists().where(
                model_class.id == item_id)).scalar():
            return make_response(open(
                'angular_flask/templates/index.html').read())
    abort(404)

# special file handlers and error handlers
@app.route('/favicon.ico')
def favicon():
    return send_from_directory(os.path.join(app.root_path, 'static'),
                               'img/favicon.ico')

@app.errorhandler(404)
def page_not_found(e):
    return render_template('404.html'), 404

<|endoftext|>


<commit_before>"""Fix missing logical file for composite resources. If there are resource files in django
for any composite resource that are not part of any logical file, each of those files are made
part of a generic logical file.

* By default, prints errors on stdout.
* Optional argument --log: logs output to system log.
"""

from django.core.management.base import BaseCommand
from hs_core.models import BaseResource

class Command(BaseCommand):
    help = "Set generic logical file for any resource file that is not part of any logical file."

    def add_arguments(self, parser):

        # a list of resource id's, or none to check all resources
        parser.add_argument('resource_ids', nargs='*', type=str)

        # Named (optional) arguments
        parser.add_argument(
            '--log',
            action='store_true',  # True for presence, False for absence
            dest='log',           # value is options['log']
            help='log errors to system log',
        )

    def handle(self, *args, **options):
        if len(options['resource_ids']) > 0:  # an array of resource short_id to check.
            for rid in options['resource_ids']:
                try:
                    resource = BaseResource.objects.get(short_id=rid)
                except BaseResource.DoesNotExist:
                    msg = "Resource with id {} not found in Django Resources".format(rid)
                    print(msg)
                    continue
                if resource.resource_type != 'CompositeResource':
                    print("resource {} has type {}: skipping".format(resource.short_id,
                                                                     resource.resource_type))
                    continue

                print("SETTING GENERIC LOGICAL FILE FOR FILES IN RESOURCE {}".format(rid))
                for res_file in resource.files.all():
                    if not res_file.has_logical_file:
                        print("Logical file missing for file {}".format(res_file.short_path))
                resource.set_default_logical_file()

        else:  # check all composite resources
            print("SETTING GENERIC LOGICAL FILE FOR FILES IN ALL COMPOSITE RESOURCES")
            for r in BaseResource.objects.filter(resource_type="CompositeResource"):
                print("SETTING GENERIC LOGICAL FILE FOR FILES IN RESOURCE {}".format(r.short_id))
                for res_file in r.files.all():
                    if not res_file.has_logical_file:
                        print("Logical file missing for file {}".format(res_file.short_path))
                r.set_default_logical_file()
<commit_msg>[#2457] bug fix<commit_after>"""Fix missing logical file for composite resources. If there are resource files in django
for any composite resource that are not part of any logical file, each of those files are made
part of a generic logical file.

* By default, prints errors on stdout.
* Optional argument --log: logs output to system log.
"""

from django.core.management.base import BaseCommand

from hs_composite_resource.models import CompositeResource

class Command(BaseCommand):
    help = "Set generic logical file for any resource file that is not part of any logical file."

    def add_arguments(self, parser):

        # a list of resource id's, or none to check all resources
        parser.add_argument('resource_ids', nargs='*', type=str)

        # Named (optional) arguments
        parser.add_argument(
            '--log',
            action='store_true',  # True for presence, False for absence
            dest='log',           # value is options['log']
            help='log errors to system log',
        )

    def handle(self, *args, **options):
        if len(options['resource_ids']) > 0:  # an array of resource short_id to check.
            for rid in options['resource_ids']:
                try:
                    resource = CompositeResource.objects.get(short_id=rid)
                except CompositeResource.DoesNotExist:
                    msg = "Resource with id {} not found in Django Resources".format(rid)
                    print(msg)
                    continue

                print("SETTING GENERIC LOGICAL FILE FOR FILES IN RESOURCE {}".format(rid))
                for res_file in resource.files.all():
                    if not res_file.has_logical_file:
                        print("Logical file missing for file {}".format(res_file.short_path))
                resource.set_default_logical_file()

        else:  # check all composite resources
            print("SETTING GENERIC LOGICAL FILE FOR FILES IN ALL COMPOSITE RESOURCES")
            for r in CompositeResource.objects.all():
                print("SETTING GENERIC LOGICAL FILE FOR FILES IN RESOURCE {}".format(r.short_id))
                for res_file in r.files.all():
                    if not res_file.has_logical_file:
                        print("Logical file missing for file {}".format(res_file.short_path))
                r.set_default_logical_file()
<|endoftext|>


<commit_before>import sublime
import sublime_plugin
import re
import string
from pprint import pprint

REGEX_ESCAPE_CHARS = '\\+*()[]{}^$?|:].,'

class JumpGroupIterator:
    '''
       given a list of region jump targets matching the given character, can emit a series of
       JumpGroup dictionaries
    '''
    def __init__(self, view, character, placeholder_chars):
        self.view = view
        self.all_jump_targets = self.find_all_jump_targets_in_visible_region(character)
        self.jump_target_index = 0
        self.placeholder_chars = placeholder_chars

    def __iter__(self):
        return self

    def has_next(self):
        return self.jump_target_index < len(self.all_jump_targets)

    def next(self):
        if not self.has_next():
            raise StopIteration

        jump_group = dict()

        for placeholder_char in self.placeholder_chars:
            if self.has_next():
                jump_group[placeholder_char] = self.all_jump_targets[self.jump_target_index]
                self.jump_target_index += 1
            else:
                break

        return jump_group

    def reset(self):
        self.jump_target_index = 0

    def find_all_jump_targets_in_visible_region(self, character):
        visible_region_begin = self.visible_region_begin()
        visible_text = self.visible_text()
        folded_regions = self.get_folded_regions(self.view)
        matching_regions = []
        escaped_character = self.escape_character(character)

        for char_at in (match.start() for match in re.finditer(escaped_character, visible_text)):
            char_point = char_at + visible_region_begin
            char_region = sublime.Region(char_point, char_point + 1)
            if not self.region_list_contains_region(folded_regions, char_region):
                matching_regions.append(char_region)

        return matching_regions

    def region_list_contains_region(self, region_list, region):

        for element_region in region_list:
            if element_region.contains(region):
                return True
        return False

    def visible_region_begin(self):
        return self.view.visible_region().begin()

    def visible_text(self):
        visible_region = self.view.visible_region()
        return self.view.substr(visible_region)

    def escape_character(self, character):
        if (REGEX_ESCAPE_CHARS.find(character) >= 0):
            return '\\' + character
        else:
            return character

    def get_folded_regions(self, view):
        '''
        No way in the API to get the folded regions without unfolding them first
        seems to be quick enough that you can't actually see them fold/unfold
        '''
        folded_regions = view.unfold(view.visible_region())
        view.fold(folded_regions)
        return folded_regions

class EasyMotionCommand(sublime_plugin.WindowCommand):
    '''
       We want a WindowCommand and not a TextComand so that we can control the edit/undo item so the user
       can't "undo" back to a state where we've transformed their selection to a-zA-Z0-9
    '''

    active_view = None
    edit = None
    jump_target_scope = None
    jump_group_iterator = None
    current_jump_group = None
    select_text = False

    def run(self, character=None, select_text=False):
        sublime.status_message("SublimeJump to " + character)

        settings = sublime.load_settings("Preferences.sublime-settings")

        self.jump_target_scope = settings.get('jump_target_scope', 'string')
        placeholder_chars = settings.get('placeholder_chars', 'abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ')
        self.active_view = self.window.active_view()
        self.select_text = select_text

        self.jump_group_iterator = JumpGroupIterator(self.active_view, character, placeholder_chars)

        if self.jump_group_iterator.has_next():
            self.prompt_for_next_jump_group()
        else:
            sublime.status_message("EasyMotion: unable to find any instances of " + character + " in visible region")

    def prompt_for_next_jump_group(self):
        if not self.jump_group_iterator.has_next():
            self.jump_group_iterator.reset()

        self.current_jump_group = self.jump_group_iterator.next()
        self.prompt_for_jump()

    def prompt_for_jump(self):
        self.activate_current_jump_group()
        try:
            self.window.show_input_panel("Pick jump target", "", self.enter_pressed, self.picked_target, self.finish_easy_motion)
        except:
            self.deactivate_current_jump_group()

    def enter_pressed(self, selection):
        # shouldn't get here unless the user hit enter, other selections should go through on_change branch
        self.prompt_for_next_jump_group()

    def picked_target(self, selection):
        if len(selection) > 0:
            # this will get used when hide_panel calls through to jump_to_winning_selection
            # can't call directly because of race condition
            self.winning_selection = self.winning_selection_from(selection)
            self.window.run_command("hide_panel")

    def winning_selection_from(self, selection):
        winning_region = None
        if selection in self.current_jump_group:
            winning_region = self.current_jump_group[selection]

        if winning_region is not None:
            if self.select_text:
                for current_selection in self.active_view.sel():
                    if winning_region.begin() < current_selection.begin():
                        return sublime.Region(winning_region.begin(), current_selection.end())
                    else:
                        return sublime.Region(current_selection.begin(), winning_region.end())
            else:
                return winning_region.begin()

    def activate_current_jump_group(self):
        '''
            Start up an edit object if we don't have one already, then create all of the jump targets
        '''
        if (self.edit is not None):
            # normally would call deactivate_current_jump_group here, but apparent ST2 bug prevents it from calling undo correctly
            # instead just decorate the new character and keep the same edit object so all changes get undone properly
            self.active_view.erase_regions("jump_match_regions")
        else:
            self.edit = self.active_view.begin_edit()

        for placeholder_char in self.current_jump_group.keys():
            self.active_view.replace(self.edit, self.current_jump_group[placeholder_char], placeholder_char)

        self.active_view.add_regions("jump_match_regions", self.current_jump_group.values(), self.jump_target_scope, "dot")

    def finish_easy_motion(self):
        '''
        We need to clean up after ourselves by restoring the view to it's original state, if the user did
        press a jump target that we've got saved, jump to it as the last action
        '''
        self.deactivate_current_jump_group()
        self.jump_to_winning_selection()

    def deactivate_current_jump_group(self):
        '''
            Close out the edit that we've been messing with and then undo it right away to return the buffer to
            the pristine state that we found it in.  Other methods ended up leaving the window in a dirty save state
            and this seems to be the cleanest way to get back to the original state
        '''
        if (self.edit is not None):
            self.active_view.end_edit(self.edit)
            self.window.run_command("undo")
            self.edit = None

        self.active_view.erase_regions("jump_match_regions")

    def jump_to_winning_selection(self):
        if self.winning_selection is not None:
            view_sel = self.active_view.sel()
            view_sel.clear()
            view_sel.add(self.winning_selection)
            self.active_view.show(self.winning_selection)
<commit_msg>fix issue #12 - cursor appearance not updated after jump (vintage mode)<commit_after>import sublime
import sublime_plugin
import re
import string
from pprint import pprint

REGEX_ESCAPE_CHARS = '\\+*()[]{}^$?|:].,'

class JumpGroupIterator:
    '''
       given a list of region jump targets matching the given character, can emit a series of
       JumpGroup dictionaries
    '''
    def __init__(self, view, character, placeholder_chars):
        self.view = view
        self.all_jump_targets = self.find_all_jump_targets_in_visible_region(character)
        self.jump_target_index = 0
        self.placeholder_chars = placeholder_chars

    def __iter__(self):
        return self

    def has_next(self):
        return self.jump_target_index < len(self.all_jump_targets)

    def next(self):
        if not self.has_next():
            raise StopIteration

        jump_group = dict()

        for placeholder_char in self.placeholder_chars:
            if self.has_next():
                jump_group[placeholder_char] = self.all_jump_targets[self.jump_target_index]
                self.jump_target_index += 1
            else:
                break

        return jump_group

    def reset(self):
        self.jump_target_index = 0

    def find_all_jump_targets_in_visible_region(self, character):
        visible_region_begin = self.visible_region_begin()
        visible_text = self.visible_text()
        folded_regions = self.get_folded_regions(self.view)
        matching_regions = []
        escaped_character = self.escape_character(character)

        for char_at in (match.start() for match in re.finditer(escaped_character, visible_text)):
            char_point = char_at + visible_region_begin
            char_region = sublime.Region(char_point, char_point + 1)
            if not self.region_list_contains_region(folded_regions, char_region):
                matching_regions.append(char_region)

        return matching_regions

    def region_list_contains_region(self, region_list, region):

        for element_region in region_list:
            if element_region.contains(region):
                return True
        return False

    def visible_region_begin(self):
        return self.view.visible_region().begin()

    def visible_text(self):
        visible_region = self.view.visible_region()
        return self.view.substr(visible_region)

    def escape_character(self, character):
        if (REGEX_ESCAPE_CHARS.find(character) >= 0):
            return '\\' + character
        else:
            return character

    def get_folded_regions(self, view):
        '''
        No way in the API to get the folded regions without unfolding them first
        seems to be quick enough that you can't actually see them fold/unfold
        '''
        folded_regions = view.unfold(view.visible_region())
        view.fold(folded_regions)
        return folded_regions

class EasyMotionCommand(sublime_plugin.WindowCommand):
    '''
       We want a WindowCommand and not a TextComand so that we can control the edit/undo item so the user
       can't "undo" back to a state where we've transformed their selection to a-zA-Z0-9
    '''

    active_view = None
    edit = None
    jump_target_scope = None
    jump_group_iterator = None
    current_jump_group = None
    select_text = False

    def run(self, character=None, select_text=False):
        sublime.status_message("SublimeJump to " + character)

        settings = sublime.load_settings("Preferences.sublime-settings")

        self.jump_target_scope = settings.get('jump_target_scope', 'string')
        placeholder_chars = settings.get('placeholder_chars', 'abcdefghijklmnopqrstuvwxyz01234567890ABCDEFGHIJKLMNOPQRSTUVWXYZ')
        self.active_view = self.window.active_view()
        self.select_text = select_text

        self.jump_group_iterator = JumpGroupIterator(self.active_view, character, placeholder_chars)

        if self.jump_group_iterator.has_next():
            self.prompt_for_next_jump_group()
        else:
            sublime.status_message("EasyMotion: unable to find any instances of " + character + " in visible region")

    def prompt_for_next_jump_group(self):
        if not self.jump_group_iterator.has_next():
            self.jump_group_iterator.reset()

        self.current_jump_group = self.jump_group_iterator.next()
        self.prompt_for_jump()

    def prompt_for_jump(self):
        self.activate_current_jump_group()
        try:
            self.window.show_input_panel("Pick jump target", "", self.enter_pressed, self.picked_target, self.finish_easy_motion)
        except:
            self.deactivate_current_jump_group()

    def enter_pressed(self, selection):
        # shouldn't get here unless the user hit enter, other selections should go through on_change branch
        self.prompt_for_next_jump_group()

    def picked_target(self, selection):
        if len(selection) > 0:
            # this will get used when hide_panel calls through to jump_to_winning_selection
            # can't call directly because of race condition
            self.winning_selection = self.winning_selection_from(selection)
            self.window.run_command("hide_panel")

    def winning_selection_from(self, selection):
        winning_region = None
        if selection in self.current_jump_group:
            winning_region = self.current_jump_group[selection]

        if winning_region is not None:
            if self.select_text:
                for current_selection in self.active_view.sel():
                    if winning_region.begin() < current_selection.begin():
                        return sublime.Region(winning_region.begin(), current_selection.end())
                    else:
                        return sublime.Region(current_selection.begin(), winning_region.end())
            else:
                return winning_region.begin()

    def activate_current_jump_group(self):
        '''
            Start up an edit object if we don't have one already, then create all of the jump targets
        '''
        if (self.edit is not None):
            # normally would call deactivate_current_jump_group here, but apparent ST2 bug prevents it from calling undo correctly
            # instead just decorate the new character and keep the same edit object so all changes get undone properly
            self.active_view.erase_regions("jump_match_regions")
        else:
            self.edit = self.active_view.begin_edit()

        for placeholder_char in self.current_jump_group.keys():
            self.active_view.replace(self.edit, self.current_jump_group[placeholder_char], placeholder_char)

        self.active_view.add_regions("jump_match_regions", self.current_jump_group.values(), self.jump_target_scope, "dot")

    def finish_easy_motion(self):
        '''
        We need to clean up after ourselves by restoring the view to it's original state, if the user did
        press a jump target that we've got saved, jump to it as the last action
        '''
        self.deactivate_current_jump_group()
        self.jump_to_winning_selection()

    def deactivate_current_jump_group(self):
        '''
            Close out the edit that we've been messing with and then undo it right away to return the buffer to
            the pristine state that we found it in.  Other methods ended up leaving the window in a dirty save state
            and this seems to be the cleanest way to get back to the original state
        '''
        if (self.edit is not None):
            self.active_view.end_edit(self.edit)
            self.window.run_command("undo")
            self.edit = None

        self.active_view.erase_regions("jump_match_regions")

    def jump_to_winning_selection(self):
        if self.winning_selection is not None:
            self.active_view.run_command('jump_to_winning_selection', {'new_sel': self.winning_selection})

class JumpToWinningSelection(sublime_plugin.TextCommand):
    def run(self, edit, new_sel):
        sel = self.view.sel()
        sel.clear()
        sel.add(long(new_sel))
<|endoftext|>