<commit_before>import sys
if '../' not in sys.path:
    sys.path.append('../')
import os
import tarfile
import shutil
import unittest
import log

class TestAppend(unittest.TestCase):

    def setUp(self):
        """call the append function before testing"""
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/test.log')

    def test_log_exists(self):
        """Check if the log file was created"""
        log = os.path.isfile('/tmp/test.log')
        self.assertTrue(log)

    def test_append(self):
        """Opens the previously created log file and checks the line"""
        log_line = open('/tmp/test.log').readline()
        info = "INFO test running a test"
        if info in log_line:
            assert True
        else:
            assert False

    def tearDown(self):
        """Delete the log file that was created for the test"""
        os.remove('/tmp/test.log')

class TestRotate(unittest.TestCase):

    def setUp(self):
        """call the append function before testing to add some lines"""
        os.mkdir('/tmp/testlog')
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')

    def test_location_verify(self):
        """Verify the location verifier"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        self.assertTrue(rotate.location_verify())

    def test_item_count(self):
        """Return the total number of files"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        expected = 1
        actual = rotate.item_count()
        self.assertEqual(actual, expected)

    def test_get_size(self):
        """Verify the correct size of a file"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        expected = 41
        actual = rotate.get_size('/tmp/testlog/test.log')
        self.assertEqual(actual, expected)

    def test_compress(self):
        """Compress a given file"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.compress('/tmp/testlog/log.tar.gz',
                '/tmp/testlog/test.log')
        try:
            tarfile.open('/tmp/testlog/log.tar.gz', 'r:gz')
            gz = True
        except ReadError:
            gz = False
        self.assertTrue(gz)

    def tearDown(self):
        """Delete the log file that was created for the test"""
        shutil.rmtree('/tmp/testlog')

if __name__ == '__main__':
    unittest.main()
<commit_msg>added tests for manager<commit_after>import sys
if '../' not in sys.path:
    sys.path.append('../')
import os
import tarfile
import shutil
import unittest
import log

class TestAppend(unittest.TestCase):

    def setUp(self):
        """call the append function before testing"""
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/test.log')

    def test_log_exists(self):
        """Check if the log file was created"""
        log = os.path.isfile('/tmp/test.log')
        self.assertTrue(log)

    def test_append(self):
        """Opens the previously created log file and checks the line"""
        log_line = open('/tmp/test.log').readline()
        info = "INFO test running a test"
        if info in log_line:
            assert True
        else:
            assert False

    def tearDown(self):
        """Delete the log file that was created for the test"""
        os.remove('/tmp/test.log')

class TestRotate(unittest.TestCase):

    def setUp(self):
        """call the append function before testing to add some lines"""
        os.mkdir('/tmp/testlog')
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')

    def test_location_verify(self):
        """Verify the location verifier"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        self.assertTrue(rotate.location_verify())

    def test_item_count(self):
        """Return the total number of files"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        expected = 1
        actual = rotate.item_count()
        self.assertEqual(actual, expected)

    def test_get_size(self):
        """Verify the correct size of a file"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        expected = 41
        actual = rotate.get_size('/tmp/testlog/test.log')
        self.assertEqual(actual, expected)

    def test_compress(self):
        """Compress a given file"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.compress('/tmp/testlog/log.tar.gz',
                '/tmp/testlog/test.log')
        try:
            tarfile.open('/tmp/testlog/log.tar.gz', 'r:gz')
            gz = True
        except ReadError:
            gz = False
        self.assertTrue(gz)

    def test_remove(self):
        """Remove a file"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.remove('/tmp/testlog/test.log')
        expected = os.path.isfile('/tmp/testlog/test.log')
        self.assertFalse(expected)

    def test_manager_first_compress(self):
        """Verify correct log rotation of 1 item"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.manager()
        expected = os.path.isfile('/tmp/testlog/test.log.1.tar.gz')
        self.assertTrue(expected)

    def test_manager_delete_log(self):
        """Verify correct log elimination when rotating"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.manager()
        expected = os.path.isfile('/tmp/testlog/test.log')
        self.assertFalse(expected)

    def test_manager_rotate(self):
        """Verify correct log rotation of 2 items"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.manager()
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')
        rotate.manager()
        expected = os.path.isfile('/tmp/testlog/test.log.2.tar.gz')
        self.assertTrue(expected)

    def test_manager_rotate_max(self):
        """Verify correct log rotation of max items"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.manager()
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')
        rotate.manager()
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')
        rotate.manager()
        expected_1 = os.path.isfile('/tmp/testlog/test.log.2.tar.gz')
        expected_2 = os.path.isfile('/tmp/testlog/test.log.1.tar.gz')

        self.assertTrue(expected_1, expected_2)

    def test_manager_rotate_max(self):
        """Verify max log files when max items is reached"""
        rotate = log.Rotate(location='/tmp/testlog',
                max_size=1,
                max_items=3,
                log_name='test.log')
        rotate.manager()
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')
        rotate.manager()
        log.append(module='test', type='INFO', line='running a test',
                log_file = '/tmp/testlog/test.log')
        rotate.manager()
        actual = rotate.item_count()
        expected = 3
        self.assertEqual(actual, expected)

    def tearDown(self):
        """Delete the log file that was created for the test"""
        shutil.rmtree('/tmp/testlog')

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from __future__ import division

import numpy as np
import six

from chainer import cuda

def _fast_hist(pred_label, gt_label, n_class):
    # Construct histogram for label evaluation.

    mask = (gt_label >= 0) & (gt_label < n_class)
    # an array of shape (n_class, n_class)
    hist = np.bincount(
        n_class * gt_label[mask].astype(int) +
        pred_label[mask], minlength=n_class**2).reshape(n_class, n_class)
    return hist

def eval_semantic_segmentation(pred_label, gt_label, n_class):
    """Evaluate results of semantic segmentation.

    This function measures four metrics: pixel accuracy,
    mean pixel accuracy, mean intersection over union and
    frequency weighted intersection over union.

    The definition of these metrics are as follows, where
    :math:`N_{ij}` is the amount of pixels of class :math:`i`
    inferred to belong to :math:`j` and there is :math:`k` classes.

    * Pixel Accuracy (PA)
        :math:`PA = \\frac
        {\\sum_{i=1}^k N_{ii}}
        {\\sum_{i=1}^k \\sum_{j=1}^k N_{ij}}`
    * Mean Pixel Accuracy (MPA)
        :math:`MPA = \\frac{1}{k}
        \\sum_{i=1}^k
        \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij}}`
    * Mean Intersection over Union (MIoU)
        :math:`MIoU = \\frac{1}{k}
        \\sum_{i=1}^k
        \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}`
    * Frequency Weighted Intersection over Union (FWIoU)
        :math:`FWIoU = \\frac{1}{\\sum_{i=1}^k \\sum_{j=1}^k N_{ij}}
        \\sum_{i=1}^k \\frac{\\sum_{j=1}^k N_{ij}N_{ii}}
        {\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}`

    The more detailed descriptions on the above metrics can be found at a
    review on semantic segmentation[1].

    .. [1] Alberto Garcia-Garcia, Sergio Orts-Escolano, Sergiu Oprea, \
    Victor Villena-Martinez, Jose Garcia-Rodriguez. \
    A Review on Deep Learning Techniques Applied to Semantic Segmentation. \
    https://arxiv.org/abs/1704.06857

    Types of :obj:`pred_label` and :obj:`gt_label` need to be same.
    The outputs are same type as the inputs.

    Args:
        pred_label (array): An integer array of image containing
            class labels as values, which is obtained from inference.
            This has shape :math:`(N, 1, H, W)` or :math:`(1, H, W)`,
            where :math:`N` is size of the batch, :math:`H` is the height
            and :math:`W` is the width.
        gt_label (array): An integer array of image containing
            the ground truth class labels as values. A pixel with value
            "-1" will be ignored during evaluation. Its shape is similar
            to :obj:`pred_label`.
            Its image size is equal to that of :obj:`pred_label`.
            This should be a one channel CHW formatted image.
        n_class (int): Number of classes.

    Returns:
        (array, array, array, array):
        A tuple of pixel accuracy, mean pixel accuracy, MIoU and FWIoU.
        These arrays have shape :math:`(N,)`, where :math:`N` is
        the number of images in the input.

    """
    xp = cuda.get_array_module(pred_label)
    pred_label = cuda.to_cpu(pred_label)
    gt_label = cuda.to_cpu(gt_label)

    ndim = pred_label.ndim
    if pred_label.ndim != gt_label.ndim:
        raise ValueError(
            'Ground truth and predicted label map should have same number '
            'of dimensions.')
    if ndim == 3:
        pred_label = pred_label[None]
        gt_label = gt_label[None]
    elif ndim < 3:
        raise ValueError('Input images need to be at least three dimensional.')

    if pred_label.shape[1] != 1 or gt_label.shape[1] != 1:
        raise ValueError('Channel sizes of inputs need to be one.')

    N = len(pred_label)
    acc = np.zeros((N,))
    acc_cls = np.zeros((N,))
    mean_iu = np.zeros((N,))
    fwavacc = np.zeros((N,))
    for i in six.moves.range(len(pred_label)):
        hist = _fast_hist(
            pred_label[i].flatten(), gt_label[i].flatten(), n_class)
        acc[i] = np.diag(hist).sum() / hist.sum()
        acc_cls_i = np.diag(hist) / hist.sum(axis=1)
        acc_cls[i] = np.nanmean(acc_cls_i)
        iu_denominator = (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
        iu = np.diag(hist) / iu_denominator
        mean_iu[i] = np.nanmean(iu)
        freq = hist.sum(axis=1) / hist.sum()
        fwavacc[i] = (freq[freq > 0] * iu[freq > 0]).sum()

    return (xp.asarray(acc), xp.asarray(acc_cls),
            xp.asarray(mean_iu), xp.asarray(fwavacc))
<commit_msg>iu --> iou<commit_after>from __future__ import division

import numpy as np
import six

from chainer import cuda

def _fast_hist(pred_label, gt_label, n_class):
    # Construct histogram for label evaluation.

    mask = (gt_label >= 0) & (gt_label < n_class)
    # an array of shape (n_class, n_class)
    hist = np.bincount(
        n_class * gt_label[mask].astype(int) +
        pred_label[mask], minlength=n_class**2).reshape(n_class, n_class)
    return hist

def eval_semantic_segmentation(pred_label, gt_label, n_class):
    """Evaluate results of semantic segmentation.

    This function measures four metrics: pixel accuracy,
    mean pixel accuracy, mean intersection over union and
    frequency weighted intersection over union.

    The definition of these metrics are as follows, where
    :math:`N_{ij}` is the amount of pixels of class :math:`i`
    inferred to belong to :math:`j` and there is :math:`k` classes.

    * Pixel Accuracy (PA)
        :math:`PA = \\frac
        {\\sum_{i=1}^k N_{ii}}
        {\\sum_{i=1}^k \\sum_{j=1}^k N_{ij}}`
    * Mean Pixel Accuracy (MPA)
        :math:`MPA = \\frac{1}{k}
        \\sum_{i=1}^k
        \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij}}`
    * Mean Intersection over Union (MIoU)
        :math:`MIoU = \\frac{1}{k}
        \\sum_{i=1}^k
        \\frac{N_{ii}}{\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}`
    * Frequency Weighted Intersection over Union (FWIoU)
        :math:`FWIoU = \\frac{1}{\\sum_{i=1}^k \\sum_{j=1}^k N_{ij}}
        \\sum_{i=1}^k \\frac{\\sum_{j=1}^k N_{ij}N_{ii}}
        {\\sum_{j=1}^k N_{ij} + \\sum_{j=1}^k N_{ji} - N_{ii}}`

    The more detailed descriptions on the above metrics can be found at a
    review on semantic segmentation[1].

    .. [1] Alberto Garcia-Garcia, Sergio Orts-Escolano, Sergiu Oprea, \
    Victor Villena-Martinez, Jose Garcia-Rodriguez. \
    A Review on Deep Learning Techniques Applied to Semantic Segmentation. \
    https://arxiv.org/abs/1704.06857

    Types of :obj:`pred_label` and :obj:`gt_label` need to be same.
    The outputs are same type as the inputs.

    Args:
        pred_label (array): An integer array of image containing
            class labels as values, which is obtained from inference.
            This has shape :math:`(N, 1, H, W)` or :math:`(1, H, W)`,
            where :math:`N` is size of the batch, :math:`H` is the height
            and :math:`W` is the width.
        gt_label (array): An integer array of image containing
            the ground truth class labels as values. A pixel with value
            "-1" will be ignored during evaluation. Its shape is similar
            to :obj:`pred_label`.
            Its image size is equal to that of :obj:`pred_label`.
            This should be a one channel CHW formatted image.
        n_class (int): Number of classes.

    Returns:
        (array, array, array, array):
        A tuple of pixel accuracy, mean pixel accuracy, MIoU and FWIoU.
        These arrays have shape :math:`(N,)`, where :math:`N` is
        the number of images in the input.

    """
    xp = cuda.get_array_module(pred_label)
    pred_label = cuda.to_cpu(pred_label)
    gt_label = cuda.to_cpu(gt_label)

    ndim = pred_label.ndim
    if pred_label.ndim != gt_label.ndim:
        raise ValueError(
            'Ground truth and predicted label map should have same number '
            'of dimensions.')
    if ndim == 3:
        pred_label = pred_label[None]
        gt_label = gt_label[None]
    elif ndim < 3:
        raise ValueError('Input images need to be at least three dimensional.')

    if pred_label.shape[1] != 1 or gt_label.shape[1] != 1:
        raise ValueError('Channel sizes of inputs need to be one.')

    N = len(pred_label)
    acc = np.zeros((N,))
    acc_cls = np.zeros((N,))
    mean_iou = np.zeros((N,))
    fwavacc = np.zeros((N,))
    for i in six.moves.range(len(pred_label)):
        hist = _fast_hist(
            pred_label[i].flatten(), gt_label[i].flatten(), n_class)
        acc[i] = np.diag(hist).sum() / hist.sum()
        acc_cls_i = np.diag(hist) / hist.sum(axis=1)
        acc_cls[i] = np.nanmean(acc_cls_i)
        iou_denominator = (hist.sum(axis=1) + hist.sum(axis=0) - np.diag(hist))
        iou = np.diag(hist) / iou_denominator
        mean_iou[i] = np.nanmean(iou)
        freq = hist.sum(axis=1) / hist.sum()
        fwavacc[i] = (freq[freq > 0] * iou[freq > 0]).sum()

    return (xp.asarray(acc), xp.asarray(acc_cls),
            xp.asarray(mean_iou), xp.asarray(fwavacc))
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Parse and translate an EBNF grammar into a Python parser for
the described language.
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import codecs
import argparse
import os
import pickle
import sys

from .exceptions import GrakoException
from .parser import GrakoGrammarGenerator

DESCRIPTION = ('GRAKO (for "grammar compiler") takes grammars'
               ' in a variation of EBNF as input, and outputs a memoizing'
               ' PEG/Packrat parser in Python.'
               )

argparser = argparse.ArgumentParser(prog='grako',
                                    description=DESCRIPTION
                                    )
argparser.add_argument('-b', '--binary',
                       help='generate a pickled grammar model (requires --output)',
                       action='store_true'
                       )
argparser.add_argument('-d', '--draw',
                       help='generate a diagram of the grammar (requires --output)',
                       action='store_true'
                       )
argparser.add_argument('filename',
                       metavar='GRAMMAR',
                       help='The filename of the Grako grammar'
                       )
argparser.add_argument('-n', '--no-nameguard',
                       help='allow tokens that are prefixes of others',
                       dest="nameguard", action='store_false', default=True
                       )
argparser.add_argument('-m', '--name',
                       nargs=1,
                       metavar='NAME',
                       help='Name for the grammar (defaults to GRAMMAR base name)'
                       )
argparser.add_argument('-o', '--output',
                       metavar='FILE',
                       help='output file (default is stdout)'
                       )
argparser.add_argument('-p', '--pretty',
                       help='prettify the input grammar',
                       action='store_true'
                       )
argparser.add_argument('-t', '--trace',
                       help='produce verbose parsing output',
                       action='store_true'
                       )
argparser.add_argument('-w', '--ws',
                       metavar='CHARACTERS',
                       help='characters to skip during parsing (use "" to disable)'
                       )

def genmodel(name, grammar, trace=False, filename=None):
    parser = GrakoGrammarGenerator(name, trace=trace)
    return parser.parse(grammar, filename=filename)

def gencode(name, grammar, trace=False, filename=None):
    model = genmodel(name, grammar, trace=trace, filename=filename)
    return model.render()

def main():
    import logging
    logging.basicConfig(format='%(levelname)s: %(message)s')
    log = logging.getLogger()
    try:
        args = argparser.parse_args()
    except Exception as e:
        print(e)
        sys.exit(1)

    binary = args.binary
    filename = args.filename
    name = args.name
    nameguard = args.nameguard
    draw = args.draw
    outfile = args.outfile
    pretty = args.pretty
    trace = args.trace
    whitespace = args.whitespace

    if binary and not outfile:
        log.error('--binary requires --outfile')
        sys.exit(1)

    if draw and not outfile:
        log.error('--draw requires --outfile')
        sys.exit(1)

    if sum((binary, draw, pretty)) > 1:
        log.error('either --binary or --draw or --pretty')
        sys.exit(1)

    if name is None:
        name = os.path.splitext(os.path.basename(filename))[0]

    if outfile and os.path.isfile(outfile):
        os.unlink(outfile)

    grammar = codecs.open(filename, 'r', encoding='utf-8').read()

    if outfile:
        dirname = os.path.dirname(outfile)
        if dirname and not os.path.isdir(dirname):
            os.makedirs(dirname)

    try:
        model = genmodel(name, grammar, trace=trace, filename=filename)
        model.whitespace = repr(whitespace)
        model.nameguard = repr(nameguard)

        if binary:
            result = pickle.dumps(model, protocol=2)
        elif pretty:
            result = str(model)
        else:
            result = model.render()

        if draw:
            from . import diagrams
            diagrams.draw(outfile, model)
        elif outfile:
            with codecs.open(outfile, 'w', encoding='utf-8') as f:
                f.write(result)
        else:
            print(result)
    except GrakoException as e:
        log.error(str(e))

if __name__ == '__main__':
    main()
<commit_msg>BUG: Using incorrect names for commandline parameters.<commit_after># -*- coding: utf-8 -*-
"""
Parse and translate an EBNF grammar into a Python parser for
the described language.
"""
from __future__ import (absolute_import, division, print_function,
                        unicode_literals)
import codecs
import argparse
import os
import pickle
import sys

from .exceptions import GrakoException
from .parser import GrakoGrammarGenerator

DESCRIPTION = ('GRAKO (for "grammar compiler") takes grammars'
               ' in a variation of EBNF as input, and outputs a memoizing'
               ' PEG/Packrat parser in Python.'
               )

argparser = argparse.ArgumentParser(prog='grako',
                                    description=DESCRIPTION
                                    )
argparser.add_argument('-b', '--binary',
                       help='generate a pickled grammar model (requires --output)',
                       action='store_true'
                       )
argparser.add_argument('-d', '--draw',
                       help='generate a diagram of the grammar (requires --output)',
                       action='store_true'
                       )
argparser.add_argument('filename',
                       metavar='GRAMMAR',
                       help='The filename of the Grako grammar'
                       )
argparser.add_argument('-n', '--no-nameguard',
                       help='allow tokens that are prefixes of others',
                       dest="nameguard", action='store_false', default=True
                       )
argparser.add_argument('-m', '--name',
                       nargs=1,
                       metavar='NAME',
                       help='Name for the grammar (defaults to GRAMMAR base name)'
                       )
argparser.add_argument('-o', '--output',
                       metavar='FILE',
                       help='output file (default is stdout)'
                       )
argparser.add_argument('-p', '--pretty',
                       help='prettify the input grammar',
                       action='store_true'
                       )
argparser.add_argument('-t', '--trace',
                       help='produce verbose parsing output',
                       action='store_true'
                       )
argparser.add_argument('-w', '--whitespace',
                       metavar='CHARACTERS',
                       help='characters to skip during parsing (use "" to disable)'
                       )

def genmodel(name, grammar, trace=False, filename=None):
    parser = GrakoGrammarGenerator(name, trace=trace)
    return parser.parse(grammar, filename=filename)

def gencode(name, grammar, trace=False, filename=None):
    model = genmodel(name, grammar, trace=trace, filename=filename)
    return model.render()

def main():
    import logging
    logging.basicConfig(format='%(levelname)s: %(message)s')
    log = logging.getLogger()
    try:
        args = argparser.parse_args()
    except Exception as e:
        print(e)
        sys.exit(1)

    binary = args.binary
    filename = args.filename
    name = args.name
    nameguard = args.nameguard
    draw = args.draw
    outfile = args.output
    pretty = args.pretty
    trace = args.trace
    whitespace = args.whitespace

    if binary and not outfile:
        log.error('--binary requires --outfile')
        sys.exit(1)

    if draw and not outfile:
        log.error('--draw requires --outfile')
        sys.exit(1)

    if sum((binary, draw, pretty)) > 1:
        log.error('either --binary or --draw or --pretty')
        sys.exit(1)

    if name is None:
        name = os.path.splitext(os.path.basename(filename))[0]

    if outfile and os.path.isfile(outfile):
        os.unlink(outfile)

    grammar = codecs.open(filename, 'r', encoding='utf-8').read()

    if outfile:
        dirname = os.path.dirname(outfile)
        if dirname and not os.path.isdir(dirname):
            os.makedirs(dirname)

    try:
        model = genmodel(name, grammar, trace=trace, filename=filename)
        model.whitespace = repr(whitespace)
        model.nameguard = repr(nameguard)

        if binary:
            result = pickle.dumps(model, protocol=2)
        elif pretty:
            result = str(model)
        else:
            result = model.render()

        if draw:
            from . import diagrams
            diagrams.draw(outfile, model)
        elif outfile:
            with codecs.open(outfile, 'w', encoding='utf-8') as f:
                f.write(result)
        else:
            print(result)
    except GrakoException as e:
        log.error(str(e))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import os
import csv
import json

def get_cut_value(args, cut, cut_hash, pivotIndex = 0):
  filename = os.path.join(args.outputHash, '{0}.json'.format(cut_hash))
  if not os.path.exists(filename): return 0
  val = 0
  with open(filename) as f:
    cuts = json.load(f)
    found_cut = False
    for entry in cuts:
      if entry['selections'] == cut:
        found_cut = True
        break
    if found_cut:
      val = entry['pivot'][pivotIndex]
    else:
      print 'Did not find cut '+cut+' in hash file'
      val = -1
  return val

def nbinsx(args):
    return int((args.g_max - args.g_min) / args.bin_size)

def nbinsy(args):
    return int((args.l_max - args.l_min) / args.bin_size)

def init_canvas(args):

    #gStyle.SetPalette(1);

    c = ROOT.TCanvas("c", "", 0, 0, args.x_dim, args.y_dim)
    c.SetRightMargin(0.16)
    c.SetTopMargin(0.07)

    return c

def axis_labels(args,cut):

    return ";m_{#tilde{g}} [GeV]; m_{#tilde{#chi}^{0}_{1}} [GeV];%s" % cut

def init_hist(args, supercut, pivotIndex = 0):
    numPivots = len(supercut['st3'])
    formattedCut = supercut['selections'].format(*(['#']*pivotIndex + ['?'] + ['#']*(numPivots - 1 - pivotIndex)))
    return ROOT.TH2F("grid",
                axis_labels(args,formattedCut),
                nbinsx(args),
                args.g_min,
                args.g_max,
                nbinsy(args),
                args.l_min,
                args.l_max)

def draw_hist(hist, nSigs=1):
    hist.SetMarkerSize(800)
    hist.SetMarkerColor(ROOT.kWhite)
    #gStyle.SetPalette(51)
    ROOT.gStyle.SetPaintTextFormat("1.{0:d}f".format(nSigs));
    hist.Draw("TEXT45 COLZ")

def draw_labels(lumi):
    txt = ROOT.TLatex()
    txt.SetNDC()
    txt.DrawText(0.32,0.87,"Internal")
    txt.DrawText(0.2,0.82,"Simulation")
    #txt.SetTextSize(0.030)
    txt.SetTextSize(18)
    txt.DrawLatex(0.16,0.95,"#tilde{g}-#tilde{g} production, #tilde{g} #rightarrow t #bar{t} + #tilde{#chi}^{0}_{1}")
    txt.DrawLatex(0.62,0.95,"L_{int} = %d fb^{-1}, #sqrt{s} = 13 TeV"% lumi)
    txt.SetTextFont(72)
    txt.SetTextSize(0.05)
    txt.DrawText(0.2,0.87,"ATLAS")
    txt.SetTextFont(12)
    txt.SetTextAngle(38)
    txt.SetTextSize(0.02)
    txt.DrawText(0.33,0.63,"Kinematically Forbidden")

def draw_text(path):

    if path is None:
        return

    txt = ROOT.TLatex()
    txt.SetNDC()
    txt.SetTextSize(0.030)

    with open(path,'r') as f:
        reader = csv.reader(f,delimiter=",")
        for row in reader:
            txt.DrawLatex(float(row[0]), float(row[1]), row[2])

def draw_line(topmass=173.34):
  l=ROOT.TLine(1000,1000,2000,2000)
  l.SetLineStyle(2)
  l.DrawLine(args.g_min,args.g_min-2*topmass,args.l_max+2*topmass,args.l_max)

from array import *
def exclusion():
  x = array('d',[1400,1600,1600,1400])
  y = array('d',[600,600,800,600])
  p=TPolyLine(4,x,y)
  p.SetFillColor(1)
  p.SetFillStyle(3001)
  #p.DrawPolyLine(4,x,y)
  return p

if __name__ == '__main__':
  import argparse
  import subprocess

  class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter):
    pass

  __version__ = subprocess.check_output(["git", "describe", "--always"], cwd=os.path.dirname(os.path.realpath(__file__))).strip()
  __short_hash__ = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=os.path.dirname(os.path.realpath(__file__))).strip()

  parser = argparse.ArgumentParser(description='Author: A. Cukierman, G. Stark. v.{0}'.format(__version__),
                                   formatter_class=lambda prog: CustomFormatter(prog, max_help_position=30))
  parser.add_argument('--summary', type=str, required=True, help='Summary json')
  parser.add_argument("--outputHash", type=str, required=True, help="directory where outputHash files are located")
  parser.add_argument("--supercuts", type=str, required=True, help="supercuts file detailing all selections used")
  parser.add_argument('--lumi', type=float, required=False, help='Luminosity to write on plot [ifb]', default=35)
  parser.add_argument('--text-file', type=str, required=False, help='text csv file', default=None)
  parser.add_argument('--out-directory', type=str, required=False, help='output directory', default='plots')
  parser.add_argument('-o', '--output', type=str, required=False, help='Name to put in output filenames', default='output')
  parser.add_argument('--g-min', type=float, required=False, help='Minimum gluino mass', default=800)
  parser.add_argument('--g-max', type=float, required=False, help='Maximum gluino mass', default=2500)
  parser.add_argument('--l-min', type=float, required=False, help='Minimum LSP mass', default=0)
  parser.add_argument('--l-max', type=float, required=False, help='Maximum LSP mass', default=1500)
  parser.add_argument('--bin-size', type=float, required=False, help='Size of bins to use', default=100)
  parser.add_argument('--x-dim', type=float, required=False, help='x-dimension of figure', default=800)
  parser.add_argument('--y-dim', type=float, required=False, help='y-dimension of figure', default=600)
  parser.add_argument('--top-mass', type=float, required=False, help='Mass of top quark [GeV]. Mainly meant to draw exclusion line.', default=173.34)
  parser.add_argument('--do-run1', action='store_true', help='Add Run-1 line to graph')
  parser.add_argument('--run1-color', type=int, required=False, help='Color of Run-1 line', default=46)
  parser.add_argument('--run1-excl', type=str, required=False, help='CSV file containing Run-1 exclusion points', default='run1_limit.csv')
  parser.add_argument('--run1-1sigma', type=str, required=False, help='CSV file containing Run-1 exclusion (+1 sigma) points', default='run1_limit_1sigma.csv')
  parser.add_argument('-b', '--batch', dest='batch_mode', action='store_true', help='Enable batch mode for ROOT.')

  # parse the arguments, throw errors if missing any
  args = parser.parse_args()

  import ROOT
  ROOT.gROOT.SetBatch(args.batch_mode)

  import numpy as np

  from rootpy.plotting.style import set_style, get_style
  atlas = get_style('ATLAS')
  atlas.SetPalette(51)
  set_style(atlas)

  summary = json.load(file(args.summary))

  plot_array={'sig':      [r['significance'] for r in summary],
              'signal':   [r['signal'] for r in summary],
              'bkgd':     [r['bkgd'] for r in summary],
              'mgluino':  [r['m_gluino'] for r in summary],
              'mlsp':     [r['m_lsp'] for r in summary],
              'ratio':    [r['ratio'] for r in summary]}

  # load in supercuts
  with open(args.supercuts) as f:
    supercuts = json.load(f)

  i = 0
  for supercut in supercuts:
    if supercut.get('pivot') is not None: continue
    cut = supercut['selections']
    # a cut string can have multiple pivots, need to draw a histogram for each pivot subsection
    numPivots = len(supercut['st3'])
    for pivotIndex in range(numPivots):
      print(i, cut)
      c = init_canvas(args)
      hist = init_hist(args, supercut, pivotIndex)

      for r in summary:
        g = r['m_gluino']
        l = r['m_lsp']
        z = get_cut_value(args, cut, r['hash'], pivotIndex)
        b = hist.FindFixBin(g,l)
        xx=ROOT.Long(0)
        yy=ROOT.Long(0)
        zz=ROOT.Long(0)
        hist.GetBinXYZ(b,xx,yy,zz)
        z_old =  hist.GetBinContent(xx,yy)
        newz = max(z_old,z)
        hist.SetBinContent(b,newz)
        if newz == 0:
          hist.SetBinContent(b, 0.001)

      st3 = supercut['st3'][pivotIndex]
      # number of steps
      nSteps = len(np.arange(*st3))
      hist.GetZaxis().SetRangeUser(st3[0], st3[1])
      hist.GetZaxis().CenterLabels()
      hist.GetZaxis().SetTickLength(0)
      hist.SetContour(nSteps)
      hist.GetZaxis().SetNdivisions(nSteps, False)

      draw_hist(hist, int(st3[1]-st3[0] <= 1))
      draw_labels(args.lumi)
      draw_text(args.text_file)
      draw_line(args.top_mass)
      #p = exclusion()
      #p.Draw()

      if numPivots == 1:
        savefilename = args.out_directory + '/' + args.output + '_' + str(i)
      else:
        savefilename = args.out_directory + '/' + args.output + '_' + str(i) + '-' + str(pivotIndex)

      for ext in ['pdf']:
        c.SaveAs(savefilename+'.{0}'.format(ext))
      print 'Saving file ' + savefilename
    i += 1
  print 'Done'

  exit(0)

<commit_msg>abs just in case it's negative for w/e reason<commit_after>import os
import csv
import json

def get_cut_value(args, cut, cut_hash, pivotIndex = 0):
  filename = os.path.join(args.outputHash, '{0}.json'.format(cut_hash))
  if not os.path.exists(filename): return 0
  val = 0
  with open(filename) as f:
    cuts = json.load(f)
    found_cut = False
    for entry in cuts:
      if entry['selections'] == cut:
        found_cut = True
        break
    if found_cut:
      val = entry['pivot'][pivotIndex]
    else:
      print 'Did not find cut '+cut+' in hash file'
      val = -1
  return val

def nbinsx(args):
    return int((args.g_max - args.g_min) / args.bin_size)

def nbinsy(args):
    return int((args.l_max - args.l_min) / args.bin_size)

def init_canvas(args):

    #gStyle.SetPalette(1);

    c = ROOT.TCanvas("c", "", 0, 0, args.x_dim, args.y_dim)
    c.SetRightMargin(0.16)
    c.SetTopMargin(0.07)

    return c

def axis_labels(args,cut):

    return ";m_{#tilde{g}} [GeV]; m_{#tilde{#chi}^{0}_{1}} [GeV];%s" % cut

def init_hist(args, supercut, pivotIndex = 0):
    numPivots = len(supercut['st3'])
    formattedCut = supercut['selections'].format(*(['#']*pivotIndex + ['?'] + ['#']*(numPivots - 1 - pivotIndex)))
    return ROOT.TH2F("grid",
                axis_labels(args,formattedCut),
                nbinsx(args),
                args.g_min,
                args.g_max,
                nbinsy(args),
                args.l_min,
                args.l_max)

def draw_hist(hist, nSigs=1):
    hist.SetMarkerSize(800)
    hist.SetMarkerColor(ROOT.kWhite)
    #gStyle.SetPalette(51)
    ROOT.gStyle.SetPaintTextFormat("1.{0:d}f".format(nSigs));
    hist.Draw("TEXT45 COLZ")

def draw_labels(lumi):
    txt = ROOT.TLatex()
    txt.SetNDC()
    txt.DrawText(0.32,0.87,"Internal")
    txt.DrawText(0.2,0.82,"Simulation")
    #txt.SetTextSize(0.030)
    txt.SetTextSize(18)
    txt.DrawLatex(0.16,0.95,"#tilde{g}-#tilde{g} production, #tilde{g} #rightarrow t #bar{t} + #tilde{#chi}^{0}_{1}")
    txt.DrawLatex(0.62,0.95,"L_{int} = %d fb^{-1}, #sqrt{s} = 13 TeV"% lumi)
    txt.SetTextFont(72)
    txt.SetTextSize(0.05)
    txt.DrawText(0.2,0.87,"ATLAS")
    txt.SetTextFont(12)
    txt.SetTextAngle(38)
    txt.SetTextSize(0.02)
    txt.DrawText(0.33,0.63,"Kinematically Forbidden")

def draw_text(path):

    if path is None:
        return

    txt = ROOT.TLatex()
    txt.SetNDC()
    txt.SetTextSize(0.030)

    with open(path,'r') as f:
        reader = csv.reader(f,delimiter=",")
        for row in reader:
            txt.DrawLatex(float(row[0]), float(row[1]), row[2])

def draw_line(topmass=173.34):
  l=ROOT.TLine(1000,1000,2000,2000)
  l.SetLineStyle(2)
  l.DrawLine(args.g_min,args.g_min-2*topmass,args.l_max+2*topmass,args.l_max)

from array import *
def exclusion():
  x = array('d',[1400,1600,1600,1400])
  y = array('d',[600,600,800,600])
  p=TPolyLine(4,x,y)
  p.SetFillColor(1)
  p.SetFillStyle(3001)
  #p.DrawPolyLine(4,x,y)
  return p

if __name__ == '__main__':
  import argparse
  import subprocess

  class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter):
    pass

  __version__ = subprocess.check_output(["git", "describe", "--always"], cwd=os.path.dirname(os.path.realpath(__file__))).strip()
  __short_hash__ = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], cwd=os.path.dirname(os.path.realpath(__file__))).strip()

  parser = argparse.ArgumentParser(description='Author: A. Cukierman, G. Stark. v.{0}'.format(__version__),
                                   formatter_class=lambda prog: CustomFormatter(prog, max_help_position=30))
  parser.add_argument('--summary', type=str, required=True, help='Summary json')
  parser.add_argument("--outputHash", type=str, required=True, help="directory where outputHash files are located")
  parser.add_argument("--supercuts", type=str, required=True, help="supercuts file detailing all selections used")
  parser.add_argument('--lumi', type=float, required=False, help='Luminosity to write on plot [ifb]', default=35)
  parser.add_argument('--text-file', type=str, required=False, help='text csv file', default=None)
  parser.add_argument('--out-directory', type=str, required=False, help='output directory', default='plots')
  parser.add_argument('-o', '--output', type=str, required=False, help='Name to put in output filenames', default='output')
  parser.add_argument('--g-min', type=float, required=False, help='Minimum gluino mass', default=800)
  parser.add_argument('--g-max', type=float, required=False, help='Maximum gluino mass', default=2500)
  parser.add_argument('--l-min', type=float, required=False, help='Minimum LSP mass', default=0)
  parser.add_argument('--l-max', type=float, required=False, help='Maximum LSP mass', default=1500)
  parser.add_argument('--bin-size', type=float, required=False, help='Size of bins to use', default=100)
  parser.add_argument('--x-dim', type=float, required=False, help='x-dimension of figure', default=800)
  parser.add_argument('--y-dim', type=float, required=False, help='y-dimension of figure', default=600)
  parser.add_argument('--top-mass', type=float, required=False, help='Mass of top quark [GeV]. Mainly meant to draw exclusion line.', default=173.34)
  parser.add_argument('--do-run1', action='store_true', help='Add Run-1 line to graph')
  parser.add_argument('--run1-color', type=int, required=False, help='Color of Run-1 line', default=46)
  parser.add_argument('--run1-excl', type=str, required=False, help='CSV file containing Run-1 exclusion points', default='run1_limit.csv')
  parser.add_argument('--run1-1sigma', type=str, required=False, help='CSV file containing Run-1 exclusion (+1 sigma) points', default='run1_limit_1sigma.csv')
  parser.add_argument('-b', '--batch', dest='batch_mode', action='store_true', help='Enable batch mode for ROOT.')

  # parse the arguments, throw errors if missing any
  args = parser.parse_args()

  import ROOT
  ROOT.gROOT.SetBatch(args.batch_mode)

  import numpy as np

  from rootpy.plotting.style import set_style, get_style
  atlas = get_style('ATLAS')
  atlas.SetPalette(51)
  set_style(atlas)

  summary = json.load(file(args.summary))

  plot_array={'sig':      [r['significance'] for r in summary],
              'signal':   [r['signal'] for r in summary],
              'bkgd':     [r['bkgd'] for r in summary],
              'mgluino':  [r['m_gluino'] for r in summary],
              'mlsp':     [r['m_lsp'] for r in summary],
              'ratio':    [r['ratio'] for r in summary]}

  # load in supercuts
  with open(args.supercuts) as f:
    supercuts = json.load(f)

  i = 0
  for supercut in supercuts:
    if supercut.get('pivot') is not None: continue
    cut = supercut['selections']
    # a cut string can have multiple pivots, need to draw a histogram for each pivot subsection
    numPivots = len(supercut['st3'])
    for pivotIndex in range(numPivots):
      print(i, cut)
      c = init_canvas(args)
      hist = init_hist(args, supercut, pivotIndex)

      for r in summary:
        g = r['m_gluino']
        l = r['m_lsp']
        z = get_cut_value(args, cut, r['hash'], pivotIndex)
        b = hist.FindFixBin(g,l)
        xx=ROOT.Long(0)
        yy=ROOT.Long(0)
        zz=ROOT.Long(0)
        hist.GetBinXYZ(b,xx,yy,zz)
        z_old =  hist.GetBinContent(xx,yy)
        newz = max(z_old,z)
        hist.SetBinContent(b,newz)
        if newz == 0:
          hist.SetBinContent(b, 0.001)

      st3 = supercut['st3'][pivotIndex]
      # number of steps
      nSteps = len(np.arange(*st3))
      hist.GetZaxis().SetRangeUser(st3[0], st3[1])
      hist.GetZaxis().CenterLabels()
      hist.GetZaxis().SetTickLength(0)
      hist.SetContour(nSteps)
      hist.GetZaxis().SetNdivisions(nSteps, False)

      draw_hist(hist, int(abs(st3[1]-st3[0] <= 1)))
      draw_labels(args.lumi)
      draw_text(args.text_file)
      draw_line(args.top_mass)
      #p = exclusion()
      #p.Draw()

      if numPivots == 1:
        savefilename = args.out_directory + '/' + args.output + '_' + str(i)
      else:
        savefilename = args.out_directory + '/' + args.output + '_' + str(i) + '-' + str(pivotIndex)

      for ext in ['pdf']:
        c.SaveAs(savefilename+'.{0}'.format(ext))
      print 'Saving file ' + savefilename
    i += 1
  print 'Done'

  exit(0)

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Implementation of a Linked Data Context structure based on the JSON-LD
definition of contexts. See:

    http://json-ld.org/

"""

# from __future__ import with_statement
from urlparse import urljoin
try:
    import json
    assert json  # workaround for pyflakes issue #13
except ImportError:
    import simplejson as json

from rdflib.namespace import RDF, split_uri
from rdflib.parser import create_input_source
from rdflib.py3compat import PY3
if PY3:
    from io import StringIO

RDF_TYPE = unicode(RDF.type)

CONTEXT_KEY = '@context'
LANG_KEY = '@language'
ID_KEY = '@id'
TYPE_KEY = '@type'
VALUE_KEY = '@value'
LIST_KEY = '@list'
CONTAINER_KEY = '@container'
SET_KEY = '@set'
REV_KEY = '@rev'  # EXPERIMENTAL
GRAPH_KEY = '@graph'
VOCAB_KEY = '@vocab'
KEYS = set(
    [LANG_KEY, ID_KEY, TYPE_KEY, VALUE_KEY, LIST_KEY, REV_KEY, GRAPH_KEY])

class Context(object):

    def __init__(self, source=None):
        self._key_map = {}
        self._iri_map = {}
        self._term_map = {}
        self.lang = None
        self.vocab = None
        if source:
            self.load(source)

    terms = property(lambda self: self._term_map.values())

    context_key = CONTEXT_KEY
    lang_key = property(lambda self: self._key_map.get(LANG_KEY, LANG_KEY))
    id_key = property(lambda self: self._key_map.get(ID_KEY, ID_KEY))
    type_key = property(lambda self: self._key_map.get(TYPE_KEY, TYPE_KEY))
    value_key = property(lambda self: self._key_map.get(VALUE_KEY, VALUE_KEY))
    list_key = property(lambda self: self._key_map.get(LIST_KEY, LIST_KEY))
    container_key = CONTAINER_KEY
    set_key = SET_KEY
    rev_key = property(lambda self: self._key_map.get(REV_KEY, REV_KEY))
    graph_key = property(lambda self: self._key_map.get(GRAPH_KEY, GRAPH_KEY))

    def load(self, source, base=None, visited_urls=None):
        if CONTEXT_KEY in source:
            source = source[CONTEXT_KEY]
        if isinstance(source, list):
            sources = source
        else:
            sources = [source]
        for data in sources:
            if isinstance(data, basestring):
                url = urljoin(base, data)
                visited_urls = visited_urls or []
                visited_urls.append(url)
                sub_defs = source_to_json(url)
                self.load(sub_defs, url, visited_urls)
                continue
            self.lang = data.get(LANG_KEY, self.lang)
            self.vocab = data.get(VOCAB_KEY, self.vocab)
            for key, value in data.items():
                if key in (LANG_KEY, VOCAB_KEY):
                    continue
                elif isinstance(value, unicode) and value in KEYS:
                    self._key_map[value] = key
                else:
                    term = self._create_term(data, key, value)
                    self.add_term(term)

    def _create_term(self, data, key, dfn):
        if isinstance(dfn, dict):
            if ID_KEY not in dfn and self.vocab:
                iri = self.vocab + key
            else:
                iri = self._rec_expand(data, dfn.get(ID_KEY))
            type_val = dfn.get(TYPE_KEY)
            if type_val in (ID_KEY, TYPE_KEY):
                coercion = type_val
            else:
                coercion = self._rec_expand(data, type_val)
            return Term(iri, key, coercion, dfn.get(CONTAINER_KEY))
        else:
            iri = self._rec_expand(data, dfn)
            return Term(iri, key)

    def _rec_expand(self, data, expr, prev=None):
        if expr == prev:
            return expr
        is_term, pfx, nxt = self._prep_expand(expr)
        if is_term and self.vocab:
            return self.vocab + expr
        if pfx:
            iri = data.get(pfx) or self.expand(pfx)
            nxt = iri + nxt
        return self._rec_expand(data, nxt, expr)

    def _prep_expand(self, expr):
        if ':' not in expr:
            return True, None, expr
        pfx, local = expr.split(':', 1)
        if not local.startswith('//'):
            return False, pfx, local
        else:
            return False, None, expr

    def add_term(self, term):
        self._iri_map[term.iri] = term
        self._term_map[term.key] = term

    def get_term(self, iri):
        return self._iri_map.get(iri)

    def shrink(self, iri):
        iri = unicode(iri)
        term = self._iri_map.get(iri)
        if term:
            return term.key
        if iri == RDF_TYPE:
            # NOTE: only if no term for the rdf:type IRI is defined
            return self.type_key
        try:
            ns, name = split_uri(iri)
            term = self._iri_map.get(ns)
            if term:
                return ":".join((term.key, name))
            elif ns == self.vocab:
                return name
        except:
            pass
        return iri

    def expand(self, term_curie_or_iri):
        term_curie_or_iri = unicode(term_curie_or_iri)
        is_term, pfx, local = self._prep_expand(term_curie_or_iri)
        # TODO: is empty string pfx (test/tests/rdf-0009.jsonld) really ok?
        #if pfx:
        if pfx is not None:
            ns = self._term_map.get(pfx)
            if ns and ns.iri:
                return ns.iri + local
        else:
            term = self._term_map.get(term_curie_or_iri)
            if term:
                return term.iri
            elif self.vocab and ':' not in term_curie_or_iri:
                return self.vocab + term_curie_or_iri
        return term_curie_or_iri

    def to_dict(self):
        data = {}
        if self.lang:
            data[LANG_KEY] = self.lang
        if self.vocab:
            data[VOCAB_KEY] = self.vocab
        for key, alias in self._key_map.items():
            if key != alias:
                data[alias] = key
        for term in self.terms:
            obj = term.iri
            if term.coercion:
                obj = {ID_KEY: term.iri}
                if term.coercion == REV_KEY:
                    obj = {REV_KEY: term.iri}
                else:
                    obj[TYPE_KEY] = term.coercion
            if term.container:
                obj[CONTAINER_KEY] = term.container
            if obj:
                data[term.key] = obj
        return data

class Term(object):
    def __init__(self, iri, key, coercion=None, container=None):
        self.iri = iri
        self.key = key
        self.coercion = coercion
        self.container = container

def source_to_json(source):
    # TODO: conneg for JSON (fix support in rdflib's URLInputSource!)
    source = create_input_source(source)

    stream = source.getByteStream()
    try:
        if PY3:
            return json.load(StringIO(stream.read().decode('utf-8')))
        else:
            return json.load(stream)
    finally:
        stream.close()
<commit_msg>Use new reverse key<commit_after># -*- coding: utf-8 -*-
"""
Implementation of a Linked Data Context structure based on the JSON-LD
definition of contexts. See:

    http://json-ld.org/

"""

# from __future__ import with_statement
from urlparse import urljoin
try:
    import json
    assert json  # workaround for pyflakes issue #13
except ImportError:
    import simplejson as json

from rdflib.namespace import RDF, split_uri
from rdflib.parser import create_input_source
from rdflib.py3compat import PY3
if PY3:
    from io import StringIO

RDF_TYPE = unicode(RDF.type)

CONTEXT_KEY = '@context'
LANG_KEY = '@language'
ID_KEY = '@id'
TYPE_KEY = '@type'
VALUE_KEY = '@value'
LIST_KEY = '@list'
CONTAINER_KEY = '@container'
SET_KEY = '@set'
REV_KEY = '@reverse'
GRAPH_KEY = '@graph'
VOCAB_KEY = '@vocab'
KEYS = set(
    [LANG_KEY, ID_KEY, TYPE_KEY, VALUE_KEY, LIST_KEY, REV_KEY, GRAPH_KEY])

class Context(object):

    def __init__(self, source=None):
        self._key_map = {}
        self._iri_map = {}
        self._term_map = {}
        self.lang = None
        self.vocab = None
        if source:
            self.load(source)

    terms = property(lambda self: self._term_map.values())

    context_key = CONTEXT_KEY
    lang_key = property(lambda self: self._key_map.get(LANG_KEY, LANG_KEY))
    id_key = property(lambda self: self._key_map.get(ID_KEY, ID_KEY))
    type_key = property(lambda self: self._key_map.get(TYPE_KEY, TYPE_KEY))
    value_key = property(lambda self: self._key_map.get(VALUE_KEY, VALUE_KEY))
    list_key = property(lambda self: self._key_map.get(LIST_KEY, LIST_KEY))
    container_key = CONTAINER_KEY
    set_key = SET_KEY
    rev_key = property(lambda self: self._key_map.get(REV_KEY, REV_KEY))
    graph_key = property(lambda self: self._key_map.get(GRAPH_KEY, GRAPH_KEY))

    def load(self, source, base=None, visited_urls=None):
        if CONTEXT_KEY in source:
            source = source[CONTEXT_KEY]
        if isinstance(source, list):
            sources = source
        else:
            sources = [source]
        for data in sources:
            if isinstance(data, basestring):
                url = urljoin(base, data)
                visited_urls = visited_urls or []
                visited_urls.append(url)
                sub_defs = source_to_json(url)
                self.load(sub_defs, url, visited_urls)
                continue
            self.lang = data.get(LANG_KEY, self.lang)
            self.vocab = data.get(VOCAB_KEY, self.vocab)
            for key, value in data.items():
                if key in (LANG_KEY, VOCAB_KEY):
                    continue
                elif isinstance(value, unicode) and value in KEYS:
                    self._key_map[value] = key
                else:
                    term = self._create_term(data, key, value)
                    self.add_term(term)

    def _create_term(self, data, key, dfn):
        if isinstance(dfn, dict):
            coercion = None
            if REV_KEY in dfn:
                iri = self._rec_expand(data, dfn.get(REV_KEY))
                coercion = REV_KEY
            elif ID_KEY not in dfn and self.vocab:
                iri = self.vocab + key
            else:
                iri = self._rec_expand(data, dfn.get(ID_KEY))
            if not coercion:
                type_val = dfn.get(TYPE_KEY)
                if type_val in (ID_KEY, TYPE_KEY):
                    coercion = type_val
                else:
                    coercion = self._rec_expand(data, type_val)
            return Term(iri, key, coercion, dfn.get(CONTAINER_KEY))
        else:
            iri = self._rec_expand(data, dfn)
            return Term(iri, key)

    def _rec_expand(self, data, expr, prev=None):
        if expr == prev:
            return expr
        is_term, pfx, nxt = self._prep_expand(expr)
        if is_term and self.vocab:
            return self.vocab + expr
        if pfx:
            iri = data.get(pfx) or self.expand(pfx)
            nxt = iri + nxt
        return self._rec_expand(data, nxt, expr)

    def _prep_expand(self, expr):
        if ':' not in expr:
            return True, None, expr
        pfx, local = expr.split(':', 1)
        if not local.startswith('//'):
            return False, pfx, local
        else:
            return False, None, expr

    def add_term(self, term):
        self._iri_map[term.iri] = term
        self._term_map[term.key] = term

    def get_term(self, iri):
        return self._iri_map.get(iri)

    def shrink(self, iri):
        iri = unicode(iri)
        term = self._iri_map.get(iri)
        if term:
            return term.key
        if iri == RDF_TYPE:
            # NOTE: only if no term for the rdf:type IRI is defined
            return self.type_key
        try:
            ns, name = split_uri(iri)
            term = self._iri_map.get(ns)
            if term:
                return ":".join((term.key, name))
            elif ns == self.vocab:
                return name
        except:
            pass
        return iri

    def expand(self, term_curie_or_iri):
        term_curie_or_iri = unicode(term_curie_or_iri)
        is_term, pfx, local = self._prep_expand(term_curie_or_iri)
        # TODO: is empty string pfx (test/tests/rdf-0009.jsonld) really ok?
        #if pfx:
        if pfx is not None:
            ns = self._term_map.get(pfx)
            if ns and ns.iri:
                return ns.iri + local
        else:
            term = self._term_map.get(term_curie_or_iri)
            if term:
                return term.iri
            elif self.vocab and ':' not in term_curie_or_iri:
                return self.vocab + term_curie_or_iri
        return term_curie_or_iri

    def to_dict(self):
        data = {}
        if self.lang:
            data[LANG_KEY] = self.lang
        if self.vocab:
            data[VOCAB_KEY] = self.vocab
        for key, alias in self._key_map.items():
            if key != alias:
                data[alias] = key
        for term in self.terms:
            obj = term.iri
            if term.coercion:
                obj = {ID_KEY: term.iri}
                if term.coercion == REV_KEY:
                    obj = {REV_KEY: term.iri}
                else:
                    obj[TYPE_KEY] = term.coercion
            if term.container:
                obj[CONTAINER_KEY] = term.container
            if obj:
                data[term.key] = obj
        return data

class Term(object):
    def __init__(self, iri, key, coercion=None, container=None):
        self.iri = iri
        self.key = key
        self.coercion = coercion
        self.container = container

def source_to_json(source):
    # TODO: conneg for JSON (fix support in rdflib's URLInputSource!)
    source = create_input_source(source)

    stream = source.getByteStream()
    try:
        if PY3:
            return json.load(StringIO(stream.read().decode('utf-8')))
        else:
            return json.load(stream)
    finally:
        stream.close()
<|endoftext|>


<commit_before>import asyncio
import functools
import inspect
import logging

from .workflow import Workflow

log = logging.getLogger(__name__)

class WorkflowInterface(object):

    workflow = None
    task = None

    def __init__(self):
        self.get_workflow()

    def get_workflow(self):
        self.task = asyncio.Task.current_task()

        for cb in self.task._callbacks:
            # inspect.getcallargs() gives acces to the implicit 'self' arg of the
            # bound method but it is marked as deprecated since Python 3.5.1
            # and the new `inspect.Signature` object does do the job :((
            if inspect.ismethod(cb):
                inst = cb.__self__
            elif isinstance(cb, functools.partial):
                try:
                    inst = cb.func.__self__
                except AttributeError:
                    continue
            else:
                continue
            if isinstance(inst, Workflow):
                self.workflow = inst

            if self.workflow is None:
                raise WorkflowNotFoundError

    def unlock_workflow_when_done(self):
        """
        Adds a done callback to the current task so as to unlock the workflow that
        handles its execution when the task gets done.
        If no workflow linked to the task can be found, raise an exception.
        It assumes all tasks scheduled from within a workflow object have at least
        one done callback which is a bound method from the workflow object.

        Note: bound methods from distinct workflow ojects in the task's
        "call when done" list are not supported. It must never happen!
        """
        self.task.add_done_callback(inst._unlock)

    def disable_children(self, children):
        """
        prevent the workflow from running children of the current tasks
        """
        log.info('disabling children %s from %s' % (children, self))
        log.warning(dir(self.task.holder))
        self.workflow.disable_children(self.task.uid, children)
<commit_msg>Minor improvements on the workflow interface<commit_after>import asyncio
import functools
import inspect
import logging

from .workflow import Workflow

log = logging.getLogger(__name__)

class WorkflowInterface(object):

    def __init__(self, task=None):
        self.task = task or asyncio.Task.current_task()
        self.workflow = self.get_workflow(self.task)

    @classmethod
    def get_workflow(cls, task):
        workflow = None
        for cb in task._callbacks:
            # inspect.getcallargs() gives acces to the implicit 'self' arg of the
            # bound method but it is marked as deprecated since Python 3.5.1
            # and the new `inspect.Signature` object does do the job :((
            if inspect.ismethod(cb):
                inst = cb.__self__
            elif isinstance(cb, functools.partial):
                try:
                    inst = cb.func.__self__
                except AttributeError:
                    continue
            else:
                continue
            if isinstance(inst, Workflow):
                workflow = inst

            if workflow is None:
                raise WorkflowNotFoundError
            return workflow

    def unlock_workflow_when_done(self):
        """
        Adds a done callback to the current task so as to unlock the workflow that
        handles its execution when the task gets done.
        If no workflow linked to the task can be found, raise an exception.
        It assumes all tasks scheduled from within a workflow object have at least
        one done callback which is a bound method from the workflow object.

        Note: bound methods from distinct workflow ojects in the task's
        "call when done" list are not supported. It must never happen!
        """
        self.task.add_done_callback(self.workflow._unlock)

    def disable_children(self, children):
        """
        prevent the workflow from running children of the current task
        """
        log.debug('disabling children %s from %s' % (children, self))
        self.workflow.disable_children(self.task.uid, children)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
"""
Code for analyzing zebrafish tracking csv output from Ctrax

Inputs:
Argument 1: File of the list of filenames to be analyzed (no extensions)
            or a single csv file output from Ctrax.
Argument 2: Filename of output CSV file
Argument 3: Mode and length (e.g., time=10 or fps=30).
            Note: time is in minutes, fps = frames per second of video

Output:
CSV file listing the percent time fish spends in different parts of tank divided
by halves and by thirds.

Author:
Justin W. Kenney
jkenney9a@gmail.com

Things to improve:
Reduce redundancy in code overall. Increase functional modularity:
1) Frame analysis
2) Analyze files (create functions for analyzing seperately csv or txt files)

This will make code more readable and easier to troubleshoot/alter down the road.

"""

import pandas as pd
import pickle #To unpickle the tank coordinates out of the .ann file
import numpy as np
import glob #For use of wild cards in getting .ann files

pd.set_option('display.precision',5)

def load_data(filename):
    """
    Input: filename

    Output: dataframe (df) object of data
    """
    return pd.read_csv(filename, header=None)

def combine_df(df):
    """
    Input: dataframe of tracking data (from CSV file)

    Output: dataframe of x and y coordinates of fish. Assumes one fish and
    combines different IDs from ctrax into one track.
    """

    coordinates = {'x':[], 'y':[]}

    for frame in df.index:

        #Get x,y coordinates of all ids present in file. The CSV output files
        #from Ctrax are setup in blocks of 6 columns where the 1st is the ID
        # and the 2nd and 3rd or the x and y coordinates, respectively.
        tracker_ids = range(0,len(df.columns),6)
        x_temp = []
        y_temp = []

        for ids in tracker_ids:
            #Ids without tracking info are "-1" in the CSV file under ID
            if df[ids][frame] >= 0:
                x_temp.append(df[ids + 1][frame])
                y_temp.append(df[ids + 2][frame])

        #Take the last of the appended coordinates (i.e, last id) and throw out
        #any spurious detections (i.e., len > 0)
        if len(x_temp) > 0:
            coordinates['x'].append(x_temp[-1])
            coordinates['y'].append(y_temp[-1])

    return pd.DataFrame(coordinates, columns = ['x','y'])

def analyze_frame_top_bottom(df, frame, half, one_third, two_thirds):
    """
    Input: dataframe of tracking data
    frame to be analyzed
    coordinates for half, one_third and two_thirds of tank

    Output: list containing where in the tank the fish was in the frame
    """
    if df['y'][frame] >= half and df['y'][frame] >= two_thirds:
        return ['top 1/2', 'top 1/3']
    elif df['y'][frame] >= half and df['y'][frame] < two_thirds:
        return ['top 1/2', 'middle 1/3']
    elif df['y'][frame] < half and df['y'][frame] >= one_third:
        return ['bottom 1/2', 'middle 1/3']
    elif df['y'][frame] < half and df['y'][frame] < one_third:
        return ['bottom 1/2', 'bottom 1/3']

def min_by_min_top_bottom_analysis(df, top, bottom, trial, mode="time"):
    """
    Input: dataframe of tracking data,
    top, bottom = coordinates for top and bottom of tank (in same space as tracking data)
    trial = the time or fps (frames per second), e.g., 5 or 30
    mode = "time" or "fps"

    Output: df of time spent in various parts of tank broken down by minute
    """

    Parameters = ['top 1/2', 'bottom 1/2', 'top 1/3', 'middle 1/3', 'bottom 1/3']

    half = ((top - bottom) / 2) + bottom
    two_thirds = (2*(top - bottom) / 3) + bottom
    one_third = ((top - bottom)/ 3) + bottom

    if mode.lower() == "time":
        trial_length = trial

        df_out = pd.DataFrame(index = Parameters,
                              columns = range(1,trial_length + 1), dtype=float)

        frames_per_min = int(len(df.index) / trial_length)

        #Find frames at each minute boundary
        frame_index = [x*frames_per_min for x in range(0,trial_length + 1)]

        for i in range(0,len(frame_index)-1):
            #Stop trial at last index;
            frames = range(frame_index[i],frame_index[i+1])
            Output = {x: 0 for x in Parameters}

            for frame in frames:
                whereabouts = analyze_frame_top_bottom(df, frame, half,
                                                       one_third, two_thirds)
                for where in whereabouts:
                    Output[where] += 1

            for x in Output.keys():
                df_out[i + 1][x] = (Output[x]/ float(len(frames))) * 100

    elif mode.lower() == "fps":
        fps = trial
        fpm = fps*60
        trial_length = float(len(df.index)) / fpm
        time_intervals = range(1,int(trial_length + 1))

        #Add on any partial minute time at end of trial
        time_intervals.append(int(trial_length) + ((trial_length % 1) * 60/100.0))
        df_out = pd.DataFrame(index = Parameters, columns = time_intervals)

        #Add 0 to beginning of time intervals to prevent index out of range errors
        time_intervals.insert(0,0)

        for t in range(0,len(time_intervals)-1):
            frames = range(time_intervals[t] * int(fpm), int(time_intervals[t+1] * fpm))
            Output = {x: 0 for x in Parameters}

            for frame in frames:
                whereabouts = analyze_frame_top_bottom(df, frame, half,
                                                       one_third, two_thirds)
                for where in whereabouts:
                    Output[where] += 1

            for x in Output.keys():
                df_out[time_intervals[t+1]][x] = (np.float64(Output[x])/ len(frames)) * 100

    return df_out

def get_top_and_bottom(ann_file):
    """
    Input: Annotation file output from Ctrax

    Output: Top, bottom, left, right coordinates for tank as dict
    """
    #Pulls out the pickled ROI object, which is a numpy array of the coordinates
    with open(ann_file, 'r') as f:
        for line in f:
            if line.find("roipolygons") != -1:
                line2 = str()
                roi_string = str()
                while "hm_cutoff" not in line2:
                    line2 = next(f)
                    roi_string += line2

    #Unpickle the sucker
    roi = pickle.loads(roi_string)

    #Get largest and smallest x and y coordinates (left/right/top/bottom of tank)
    output = {}
    output["top"] = max([roi[0][i][1] for i in range(len(roi[0]))])
    output["bottom"] = min([roi[0][i][1] for i in range(len(roi[0]))])
    output["left"] = min([roi[0][i][0] for i in range(len(roi[0]))])
    output["right"] = max([roi[0][i][0] for i in range(len(roi[0]))])

    return output

def analyze_file(files, file_type, output, mode):
    """
    Analyzes files

    Input: list of filenames, file_type = .txt or .csv, output file name,
    mode ("time=xx" or "fps=xx"),

    Output: None, writes data to the output file
    """
    if file_type == ".txt":
        f = open(sys.argv[1])
        files = [filename.strip('\n') for filename in f]
        files = [filename.strip('.csv') for filename in files]
        f.close()

    output_file = open(output,'a')

    mode_type = mode[:mode.find('=')].lower()

    if mode_type == "time":

        trial_length = int(mode[mode.find('=')+1:])

        #Add blank lines to output CSV to make it look pretty
        blanks = trial_length * " "
        blank_line = pd.DataFrame(blanks, index = [1], columns = range(trial_length))

        if file_type == ".txt":
            for filename in files:
                df = load_data(filename + ".csv")
                df = combine_df(df)
                #glob.glob returns a list; just need element of list.
                #This allows for the use of other types of movies besides .avi
                t_b = get_top_and_bottom(glob.glob(filename + ".*.ann")[0])
                df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        trial_length)
                df_out.to_csv(output_file, index_label=filename)
                blank_line.to_csv(output_file, index=False, header=False)
                print filename + " is done!"

        elif file_type == '.csv':
            df = load_data(files)
            df = combine_df(df)
            ann_file = glob.glob(files.strip('.csv') + '.*.ann')[0]
            t_b = get_top_and_bottom(ann_file)
            df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        trial_length)
            df_out.to_csv(output_file, index_label=files)
            blank_line.to_csv(output_file, index=False, header=False)
            print files + " is done!"

    elif mode_type == "fps":

        if file_type == ".txt":
            for filename in files:
                df = load_data(filename + ".csv")
                df = combine_df(df)
                fps = float(mode[mode.find('=')+1:])
                fpm = fps*60 #Calculate frames per minute
                trial_length = len(df.index)/float(fpm)

                #Add blank lines to output CSV to make it look pretty
                #Need to put it in the loop this time b/c of variable video lengths
                #using fps mode
                blanks = int(trial_length + 1) * " "
                blank_line = pd.DataFrame(blanks, index = [1],
                                          columns = range(int(trial_length)))

                t_b = get_top_and_bottom(glob.glob(filename + ".*.ann")[0])

                df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        fps, mode=mode_type)
                df_out.to_csv(output_file, index_label=filename)
                blank_line.to_csv(output_file, index=False, header=False)
                print filename + " is done!"

        elif file_type == ".csv":
            df = load_data(files)
            df = combine_df(df)
            fps = float(mode[mode.find('=')+1:])
            fpm = fps*60 #Calculate frames per minute
            trial_length = len(df.index)/float(fpm)

            #Add blank lines to output CSV to make it look pretty
            blanks = int(trial_length + 1) * " "
            blank_line = pd.DataFrame(blanks, index = [1],
                                      columns = range(int(trial_length)))

            ann_file = glob.glob(files.strip('.csv') + '.avi.ann')[0]
            t_b = get_top_and_bottom(ann_file)

            df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                    fps, mode=mode_type)
            df_out.to_csv(output_file, index_label=files)
            blank_line.to_csv(output_file, index=False, header=False)
            print files + " is done!"

    output_file.close()

if __name__ == "__main__":

    import sys

    files = sys.argv[1]
    output = sys.argv[2]
    mode = sys.argv[3]

    file_type = files[files.find('.'):].lower()

    if file_type == ".txt" or file_type == ".csv":
        analyze_file(files, file_type, output, mode)

    else:
        print "Not a supported file type."
        print "File must be a list of filenames in a .txt file or .csv output"
        print "from Ctrax."

<commit_msg>tidy up the comments<commit_after># -*- coding: utf-8 -*-
"""
Code for analyzing zebrafish tracking csv output from Ctrax

Author:
Justin W. Kenney
jkenney9a@gmail.com

Inputs:
Argument 1: File of the list of filenames to be analyzed (no extensions)
            or a single csv file output from Ctrax.
Argument 2: Filename of output CSV file
Argument 3: Mode and length (e.g., time=10 or fps=30).
            Note: time is in minutes, fps = frames per second of video

Output:
CSV file listing the percent time fish spends in different parts of tank divided
by halves and by thirds.

"""

import pandas as pd
import pickle #To unpickle the tank coordinates out of the .ann file
import numpy as np
import glob #For use of wild cards in getting .ann files

pd.set_option('display.precision',5)

def load_data(filename):
    """
    Input: filename

    Output: dataframe (df) object of data
    """
    return pd.read_csv(filename, header=None)

def combine_df(df):
    """
    Input: dataframe of tracking data (from CSV file)

    Output: dataframe of x and y coordinates of fish. Assumes one fish and
    combines different IDs from ctrax into one track.
    """

    coordinates = {'x':[], 'y':[]}

    for frame in df.index:

        #Get x,y coordinates of all ids present in file. The CSV output files
        #from Ctrax are setup in blocks of 6 columns where the 1st is the ID
        # and the 2nd and 3rd or the x and y coordinates, respectively.
        tracker_ids = range(0,len(df.columns),6)
        x_temp = []
        y_temp = []

        for ids in tracker_ids:
            #Ids without tracking info are "-1" in the CSV file under ID
            if df[ids][frame] >= 0:
                x_temp.append(df[ids + 1][frame])
                y_temp.append(df[ids + 2][frame])

        #Take the last of the appended coordinates (i.e, last id) and throw out
        #any spurious detections (i.e., len > 0)
        if len(x_temp) > 0:
            coordinates['x'].append(x_temp[-1])
            coordinates['y'].append(y_temp[-1])

    return pd.DataFrame(coordinates, columns = ['x','y'])

def analyze_frame_top_bottom(df, frame, half, one_third, two_thirds):
    """
    Input: dataframe of tracking data
    frame to be analyzed
    coordinates for half, one_third and two_thirds of tank

    Output: list containing where in the tank the fish was in the frame
    """
    if df['y'][frame] >= half and df['y'][frame] >= two_thirds:
        return ['top 1/2', 'top 1/3']
    elif df['y'][frame] >= half and df['y'][frame] < two_thirds:
        return ['top 1/2', 'middle 1/3']
    elif df['y'][frame] < half and df['y'][frame] >= one_third:
        return ['bottom 1/2', 'middle 1/3']
    elif df['y'][frame] < half and df['y'][frame] < one_third:
        return ['bottom 1/2', 'bottom 1/3']

def min_by_min_top_bottom_analysis(df, top, bottom, trial, mode="time"):
    """
    Input: dataframe of tracking data,
    top, bottom = coordinates for top and bottom of tank (in same space as tracking data)
    trial = the time or fps (frames per second), e.g., 5 or 30
    mode = "time" or "fps"

    Output: df of time spent in various parts of tank broken down by minute
    """

    Parameters = ['top 1/2', 'bottom 1/2', 'top 1/3', 'middle 1/3', 'bottom 1/3']

    half = ((top - bottom) / 2) + bottom
    two_thirds = (2*(top - bottom) / 3) + bottom
    one_third = ((top - bottom)/ 3) + bottom

    if mode.lower() == "time":
        trial_length = trial

        df_out = pd.DataFrame(index = Parameters,
                              columns = range(1,trial_length + 1), dtype=float)

        frames_per_min = int(len(df.index) / trial_length)

        #Find frames at each minute boundary
        frame_index = [x*frames_per_min for x in range(0,trial_length + 1)]

        for i in range(0,len(frame_index)-1):
            #Stop trial at last index;
            frames = range(frame_index[i],frame_index[i+1])
            Output = {x: 0 for x in Parameters}

            for frame in frames:
                whereabouts = analyze_frame_top_bottom(df, frame, half,
                                                       one_third, two_thirds)
                for where in whereabouts:
                    Output[where] += 1

            for x in Output.keys():
                df_out[i + 1][x] = (Output[x]/ float(len(frames))) * 100

    elif mode.lower() == "fps":
        fps = trial
        fpm = fps*60
        trial_length = float(len(df.index)) / fpm
        time_intervals = range(1,int(trial_length + 1))

        #Add on any partial minute time at end of trial
        time_intervals.append(int(trial_length) + ((trial_length % 1) * 60/100.0))
        df_out = pd.DataFrame(index = Parameters, columns = time_intervals)

        #Add 0 to beginning of time intervals to prevent index out of range errors
        time_intervals.insert(0,0)

        for t in range(0,len(time_intervals)-1):
            frames = range(time_intervals[t] * int(fpm), int(time_intervals[t+1] * fpm))
            Output = {x: 0 for x in Parameters}

            for frame in frames:
                whereabouts = analyze_frame_top_bottom(df, frame, half,
                                                       one_third, two_thirds)
                for where in whereabouts:
                    Output[where] += 1

            for x in Output.keys():
                df_out[time_intervals[t+1]][x] = (np.float64(Output[x])/ len(frames)) * 100

    return df_out

def get_top_and_bottom(ann_file):
    """
    Input: Annotation file output from Ctrax

    Output: Top, bottom, left, right coordinates for tank as dict
    """
    #Pulls out the pickled ROI object, which is a numpy array of the coordinates
    with open(ann_file, 'r') as f:
        for line in f:
            if line.find("roipolygons") != -1:
                line2 = str()
                roi_string = str()
                while "hm_cutoff" not in line2:
                    line2 = next(f)
                    roi_string += line2

    #Unpickle the sucker
    roi = pickle.loads(roi_string)

    #Get largest and smallest x and y coordinates (left/right/top/bottom of tank)
    output = {}
    output["top"] = max([roi[0][i][1] for i in range(len(roi[0]))])
    output["bottom"] = min([roi[0][i][1] for i in range(len(roi[0]))])
    output["left"] = min([roi[0][i][0] for i in range(len(roi[0]))])
    output["right"] = max([roi[0][i][0] for i in range(len(roi[0]))])

    return output

def analyze_file(files, file_type, output, mode):
    """
    Analyzes files

    Input: list of filenames, file_type = .txt or .csv, output file name,
    mode ("time=xx" or "fps=xx"),

    Output: None, writes data to the output file
    """
    if file_type == ".txt":
        f = open(sys.argv[1])
        files = [filename.strip('\n') for filename in f]
        files = [filename.strip('.csv') for filename in files]
        f.close()

    output_file = open(output,'a')

    mode_type = mode[:mode.find('=')].lower()

    if mode_type == "time":

        trial_length = int(mode[mode.find('=')+1:])

        #Add blank lines to output CSV to make it look pretty
        blanks = trial_length * " "
        blank_line = pd.DataFrame(blanks, index = [1], columns = range(trial_length))

        if file_type == ".txt":
            for filename in files:
                df = load_data(filename + ".csv")
                df = combine_df(df)
                #glob.glob returns a list; just need element of list.
                #This allows for the use of other types of movies besides .avi
                t_b = get_top_and_bottom(glob.glob(filename + ".*.ann")[0])
                df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        trial_length)
                df_out.to_csv(output_file, index_label=filename)
                blank_line.to_csv(output_file, index=False, header=False)
                print filename + " is done!"

        elif file_type == '.csv':
            df = load_data(files)
            df = combine_df(df)
            ann_file = glob.glob(files.strip('.csv') + '.*.ann')[0]
            t_b = get_top_and_bottom(ann_file)
            df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        trial_length)
            df_out.to_csv(output_file, index_label=files)
            blank_line.to_csv(output_file, index=False, header=False)
            print files + " is done!"

    elif mode_type == "fps":

        if file_type == ".txt":
            for filename in files:
                df = load_data(filename + ".csv")
                df = combine_df(df)
                fps = float(mode[mode.find('=')+1:])
                fpm = fps*60 #Calculate frames per minute
                trial_length = len(df.index)/float(fpm)

                #Add blank lines to output CSV to make it look pretty
                #Need to put it in the loop this time b/c of variable video lengths
                #using fps mode
                blanks = int(trial_length + 1) * " "
                blank_line = pd.DataFrame(blanks, index = [1],
                                          columns = range(int(trial_length)))

                t_b = get_top_and_bottom(glob.glob(filename + ".*.ann")[0])

                df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                        fps, mode=mode_type)
                df_out.to_csv(output_file, index_label=filename)
                blank_line.to_csv(output_file, index=False, header=False)
                print filename + " is done!"

        elif file_type == ".csv":
            df = load_data(files)
            df = combine_df(df)
            fps = float(mode[mode.find('=')+1:])
            fpm = fps*60 #Calculate frames per minute
            trial_length = len(df.index)/float(fpm)

            #Add blank lines to output CSV to make it look pretty
            blanks = int(trial_length + 1) * " "
            blank_line = pd.DataFrame(blanks, index = [1],
                                      columns = range(int(trial_length)))

            ann_file = glob.glob(files.strip('.csv') + '.avi.ann')[0]
            t_b = get_top_and_bottom(ann_file)

            df_out = min_by_min_top_bottom_analysis(df, t_b['top'], t_b['bottom'],
                                                    fps, mode=mode_type)
            df_out.to_csv(output_file, index_label=files)
            blank_line.to_csv(output_file, index=False, header=False)
            print files + " is done!"

    output_file.close()

if __name__ == "__main__":

    import sys

    files = sys.argv[1]
    output = sys.argv[2]
    mode = sys.argv[3]

    file_type = files[files.find('.'):].lower()

    if file_type == ".txt" or file_type == ".csv":
        analyze_file(files, file_type, output, mode)

    else:
        print "Not a supported file type."
        print "File must be a list of filenames in a .txt file or .csv output"
        print "from Ctrax."

<|endoftext|>


<commit_before>"""Tasks for OAuth services."""

import logging

from allauth.socialaccount.providers import registry as allauth_registry
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

from readthedocs.core.utils.tasks import PublicTask, user_id_matches
from readthedocs.oauth.notifications import (
    AttachWebhookNotification,
    InvalidProjectWebhookNotification,
)
from readthedocs.oauth.utils import SERVICE_MAP
from readthedocs.oauth.services.base import SyncServiceError
from readthedocs.projects.models import Project
from readthedocs.worker import app

from .services import registry

log = logging.getLogger(__name__)

@PublicTask.permission_check(user_id_matches)
@app.task(queue='web', base=PublicTask)
def sync_remote_repositories(user_id):
    user = User.objects.get(pk=user_id)
    failed_services = set()
    for service_cls in registry:
        for service in service_cls.for_user(user):
            try:
                service.sync()
            except SyncServiceError:
                failed_services.add(service.provider_name)
    if failed_services:
        msg = _(
            'Our access to your following accounts was revoked: {providers}. '
            'Please, reconnect them from your social account connections.'
        )
        raise Exception(
            msg.format(providers=', '.join(failed_services))
        )

@app.task(queue='web')
def attach_webhook(project_pk, user_pk, integration=None):
    """
    Add post-commit hook on project import.

    This is a brute force approach to add a webhook to a repository. We try
    all accounts until we set up a webhook. This should remain around for legacy
    connections -- that is, projects that do not have a remote repository them
    and were not set up with a VCS provider.
    """
    project = Project.objects.get(pk=project_pk)
    user = User.objects.get(pk=user_pk)
    project_notification = InvalidProjectWebhookNotification(
        context_object=project,
        user=user,
        success=False,
    )
    if integration:
        service = SERVICE_MAP.get(integration.integration_type)

        if not service:
            log.warning('There are no registered services in the application.')
            project_notification.send()
            return None
    else:
        for service_cls in registry:
            if service_cls.is_project_service(project):
                service = service_cls
                break
        else:
            log.warning('There are no registered services in the application.')
            project_notification.send()
            return None

    provider = allauth_registry.by_id(service.adapter.provider_id)
    notification = AttachWebhookNotification(
        context_object=provider,
        extra_context={'project': project},
        user=user,
        success=None,
    )

    user_accounts = service.for_user(user)
    for account in user_accounts:
        success, __ = account.setup_webhook(project, integration=integration)
        if success:
            notification.success = True
            notification.send()

            project.has_valid_webhook = True
            project.save()
            return True

    # No valid account found
    if user_accounts:
        notification.success = False
        notification.reason = AttachWebhookNotification.NO_PERMISSIONS
    else:
        notification.success = False
        notification.reason = AttachWebhookNotification.NO_ACCOUNTS

    project_notification.send()
    notification.send()
    return False
<commit_msg>OAuth: protection against deleted objects (#8081)<commit_after>"""Tasks for OAuth services."""

import logging

from allauth.socialaccount.providers import registry as allauth_registry
from django.contrib.auth.models import User
from django.utils.translation import ugettext_lazy as _

from readthedocs.core.utils.tasks import PublicTask, user_id_matches
from readthedocs.oauth.notifications import (
    AttachWebhookNotification,
    InvalidProjectWebhookNotification,
)
from readthedocs.oauth.utils import SERVICE_MAP
from readthedocs.oauth.services.base import SyncServiceError
from readthedocs.projects.models import Project
from readthedocs.worker import app

from .services import registry

log = logging.getLogger(__name__)

@PublicTask.permission_check(user_id_matches)
@app.task(queue='web', base=PublicTask)
def sync_remote_repositories(user_id):
    user = User.objects.get(pk=user_id)
    failed_services = set()
    for service_cls in registry:
        for service in service_cls.for_user(user):
            try:
                service.sync()
            except SyncServiceError:
                failed_services.add(service.provider_name)
    if failed_services:
        msg = _(
            'Our access to your following accounts was revoked: {providers}. '
            'Please, reconnect them from your social account connections.'
        )
        raise Exception(
            msg.format(providers=', '.join(failed_services))
        )

@app.task(queue='web')
def attach_webhook(project_pk, user_pk, integration=None):
    """
    Add post-commit hook on project import.

    This is a brute force approach to add a webhook to a repository. We try
    all accounts until we set up a webhook. This should remain around for legacy
    connections -- that is, projects that do not have a remote repository them
    and were not set up with a VCS provider.
    """
    project = Project.objects.filter(pk=project_pk).first()
    user = User.objects.filter(pk=user_pk).first()

    if not project or not user:
        return False

    project_notification = InvalidProjectWebhookNotification(
        context_object=project,
        user=user,
        success=False,
    )
    if integration:
        service = SERVICE_MAP.get(integration.integration_type)

        if not service:
            log.warning('There are no registered services in the application.')
            project_notification.send()
            return None
    else:
        for service_cls in registry:
            if service_cls.is_project_service(project):
                service = service_cls
                break
        else:
            log.warning('There are no registered services in the application.')
            project_notification.send()
            return None

    provider = allauth_registry.by_id(service.adapter.provider_id)
    notification = AttachWebhookNotification(
        context_object=provider,
        extra_context={'project': project},
        user=user,
        success=None,
    )

    user_accounts = service.for_user(user)
    for account in user_accounts:
        success, __ = account.setup_webhook(project, integration=integration)
        if success:
            notification.success = True
            notification.send()

            project.has_valid_webhook = True
            project.save()
            return True

    # No valid account found
    if user_accounts:
        notification.success = False
        notification.reason = AttachWebhookNotification.NO_PERMISSIONS
    else:
        notification.success = False
        notification.reason = AttachWebhookNotification.NO_ACCOUNTS

    project_notification.send()
    notification.send()
    return False
<|endoftext|>