<commit_before># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

import datasets
import datasets.pascal_voc
import os
import datasets.imdb
import xml.dom.minidom as minidom
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
import sys
from fast_rcnn.config import cfg

from utils.model import last_conv_size

class pascal_voc(datasets.imdb):
    def __init__(self, image_set, year, model_to_use='frcnn', proposal='ss', proposal_file='', devkit_path=None):
        datasets.imdb.__init__(self, 'voc_' + year + '_' + image_set)
        self._year = year
        self._image_set = image_set
        self._devkit_path = self._get_default_path() if devkit_path is None \
                            else devkit_path
        self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
        self._classes = ('__background__', # always index 0
                         'aeroplane', 'bicycle', 'bird', 'boat',
                         'bottle', 'bus', 'car', 'cat', 'chair',
                         'cow', 'diningtable', 'dog', 'horse',
                         'motorbike', 'person', 'pottedplant',
                         'sheep', 'sofa', 'train', 'tvmonitor')
        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_ext = '.jpg'
        self._image_index = self._load_image_set_index()

        self.proposal_file = proposal_file

        # Default to roidb handler
        if model_to_use == 'rpn':           # Step 1, 3
            self._roidb_handler = self.rpn_train_roidb
        elif model_to_use == 'frcnn':
            if proposal == 'rpn':           # Step 2, 4
                self._roidb_handler = self.rpn_proposal_roidb
            elif proposal == 'ss':
                self._roidb_handler = self.selective_search_roidb

        # PASCAL specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'VOCdevkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        image_path = os.path.join(self._data_path, 'JPEGImages',
                                  index + self._image_ext)
        assert os.path.exists(image_path), \
                'Path does not exist: {}'.format(image_path)
        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # Example path to image set file:
        # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
        image_set_file = os.path.join(self._data_path, 'ImageSets', 'Main',
                                      self._image_set + '.txt')
        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)
        with open(image_set_file) as f:
            image_index = [x.strip() for x in f.readlines()]
        return image_index

    def _get_default_path(self):
        """
        Return the default path where PASCAL VOC is expected to be installed.
        """
        # DJDJ
        #return os.path.join(datasets.ROOT_DIR, 'data', 'VOCdevkit' + self._year)
        return os.path.join('/home/nvidia/www/data/VOCdevkit')

    def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')

        # DJDJ
        if os.path.exists(cache_file):
        #if False:
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote gt roidb to {}'.format(cache_file)

        return gt_roidb

    def selective_search_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_selective_search_roidb.pkl')

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()
            ss_roidb = self._load_selective_search_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        else:
            roidb = self._load_selective_search_roidb(None)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote ss roidb to {}'.format(cache_file)

        return roidb

    def _load_selective_search_roidb(self, gt_roidb):
        filename = os.path.abspath(os.path.join(self.cache_path, '..',
                                                'selective_search_data',
                                                self.name + '.mat'))
        assert os.path.exists(filename), \
               'Selective search data not found at: {}'.format(filename)
        raw_data = sio.loadmat(filename)['boxes'].ravel()

        # DJDJ
        #raw_data = raw_data[:150, ]

        box_list = []
        for i in xrange(raw_data.shape[0]):
            box_list.append(raw_data[i][:, (1, 0, 3, 2)] - 1)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def selective_search_IJCV_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                '{:s}_selective_search_IJCV_top_{:d}_roidb.pkl'.
                format(self.name, self.config['top_k']))

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = self.gt_roidb()
        ss_roidb = self._load_selective_search_IJCV_roidb(gt_roidb)
        roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote ss roidb to {}'.format(cache_file)

        return roidb

    def _load_selective_search_IJCV_roidb(self, gt_roidb):
        IJCV_path = os.path.abspath(os.path.join(self.cache_path, '..',
                                                 'selective_search_IJCV_data',
                                                 'voc_' + self._year))
        assert os.path.exists(IJCV_path), \
               'Selective search IJCV data not found at: {}'.format(IJCV_path)

        top_k = self.config['top_k']
        box_list = []
        for i in xrange(self.num_images):
            filename = os.path.join(IJCV_path, self.image_index[i] + '.mat')
            raw_data = sio.loadmat(filename)
            box_list.append((raw_data['boxes'][:top_k, :]-1).astype(np.uint16))

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def rpn_train_roidb(self):
        """
        Return the database of rpn regions of interest.
        Ground-truth ROIs are also included.
        """
        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()

            roidb = gt_roidb

        return roidb

    def rpn_proposal_roidb(self):
        """
        Return the database of rpn regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_rpn_roidb.pkl')

        # DJDJ
        #if os.path.exists(cache_file):
        if False:
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} rpn roidb loaded from {}'.format(self.name, cache_file)
            return roidb
        """

        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()

            roidb = gt_roidb

            rpn_roidb = self._load_rpn_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, rpn_roidb)
        else:
            roidb = self._load_rpn_roidb(None)

        """
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote rpn roidb to {}'.format(cache_file)
        """
        return roidb

    def _load_rpn_roidb(self, gt_roidb):
        filename = os.path.abspath(self.proposal_file)
        assert os.path.exists(filename), \
               'RPN data not found at: {}'.format(filename)
        with open(filename, 'rb') as fid:
            box_list = cPickle.load(fid)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def _load_pascal_annotation(self, index):
        """
        Load image and bounding boxes info from XML file in the PASCAL VOC
        format.
        """
        filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
        # print 'Loading: {}'.format(filename)
        def get_data_from_tag(node, tag):
            return node.getElementsByTagName(tag)[0].childNodes[0].data

        with open(filename) as f:
            data = minidom.parseString(f.read())

        objs = data.getElementsByTagName('object')
        num_objs = len(objs)

        boxes = np.zeros((num_objs, 4), dtype=np.uint16)
        gt_classes = np.zeros((num_objs), dtype=np.int32)
        overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)

        # Load object bounding boxes into a data frame.
        for ix, obj in enumerate(objs):
            # Make pixel indexes 0-based
            x1 = float(get_data_from_tag(obj, 'xmin')) - 1
            y1 = float(get_data_from_tag(obj, 'ymin')) - 1
            x2 = float(get_data_from_tag(obj, 'xmax')) - 1
            y2 = float(get_data_from_tag(obj, 'ymax')) - 1
            cls = self._class_to_ind[
                    str(get_data_from_tag(obj, "name")).lower().strip()]
            boxes[ix, :] = [x1, y1, x2, y2]
            gt_classes[ix] = cls
            overlaps[ix, cls] = 1.0

        overlaps = scipy.sparse.csr_matrix(overlaps)

        return {'boxes' : boxes,
                'gt_classes': gt_classes,
                'gt_overlaps' : overlaps,
                'flipped' : False}

    def _write_voc_results_file(self, all_boxes):
        use_salt = self.config['use_salt']
        comp_id = 'comp4'
        if use_salt:
            comp_id += '-{}'.format(os.getpid())

        # VOCdevkit/results/VOC2007/Main/comp4-44503_det_test_aeroplane.txt
        path = os.path.join(self._devkit_path, 'results', 'VOC' + self._year,
                            'Main', comp_id + '_')
        for cls_ind, cls in enumerate(self.classes):
            if cls == '__background__':
                continue
            print 'Writing {} VOC results file'.format(cls)
            filename = path + 'det_' + self._image_set + '_' + cls + '.txt'
            with open(filename, 'wt') as f:
                for im_ind, index in enumerate(self.image_index):
                    dets = all_boxes[cls_ind][im_ind]
                    if dets == []:
                        continue
                    # the VOCdevkit expects 1-based indices
                    for k in xrange(dets.shape[0]):
                        f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'.
                                format(index, dets[k, -1],
                                       dets[k, 0] + 1, dets[k, 1] + 1,
                                       dets[k, 2] + 1, dets[k, 3] + 1))
        return comp_id

    def _do_matlab_eval(self, comp_id, output_dir='output'):
        rm_results = self.config['cleanup']

        path = os.path.join(os.path.dirname(__file__),
                            'VOCdevkit-matlab-wrapper')
        """
        cmd = 'cd {} && '.format(path)
        cmd += '{:s} -nodisplay -nodesktop '.format(datasets.MATLAB)
        cmd += '-r "dbstop if error; '
        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
               .format(self._devkit_path, comp_id,
                       self._image_set, output_dir, int(rm_results))
        """
        cmd = 'cd {} && '.format(path)
        cmd += '{:s} --eval "'.format(datasets.OCTAVE)
        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
               .format(self._devkit_path, comp_id,
                       self._image_set, output_dir, int(rm_results))

        print('Running:\n{}'.format(cmd))

        status = subprocess.call(cmd, shell=True)
        """
        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

        while True:
            buff = process.stdout.readline()
            if buff == '' and process.poll() != None:
                break
            sys.stdout.write(buff)
        process.wait()
        """

    def evaluate_detections(self, all_boxes, output_dir):
        comp_id = self._write_voc_results_file(all_boxes)
        self._do_matlab_eval(comp_id, output_dir)

    def competition_mode(self, on):
        if on:
            self.config['use_salt'] = False
            self.config['cleanup'] = False
        else:
            self.config['use_salt'] = True
            self.config['cleanup'] = True

if __name__ == '__main__':
    d = datasets.pascal_voc('trainval', '2007')
    res = d.roidb
    from IPython import embed; embed()
<commit_msg>Add 'label_file' in rdb of pasal voc<commit_after># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

import datasets
import datasets.pascal_voc
import os
import datasets.imdb
import xml.dom.minidom as minidom
import numpy as np
import scipy.sparse
import scipy.io as sio
import utils.cython_bbox
import cPickle
import subprocess
import sys
from fast_rcnn.config import cfg

from utils.model import last_conv_size

class pascal_voc(datasets.imdb):
    def __init__(self, image_set, year, model_to_use='frcnn', proposal='ss', proposal_file='', devkit_path=None):
        datasets.imdb.__init__(self, 'voc_' + year + '_' + image_set)
        self._year = year
        self._image_set = image_set
        self._default_path = '/home/dj/big/data/VOCdevkit'
        self._devkit_path = self._default_path if devkit_path is None \
                            else devkit_path
        self._data_path = os.path.join(self._devkit_path, 'VOC' + self._year)
        self._classes = ('__background__', # always index 0
                         'aeroplane', 'bicycle', 'bird', 'boat',
                         'bottle', 'bus', 'car', 'cat', 'chair',
                         'cow', 'diningtable', 'dog', 'horse',
                         'motorbike', 'person', 'pottedplant',
                         'sheep', 'sofa', 'train', 'tvmonitor')
        self._class_to_ind = dict(zip(self.classes, xrange(self.num_classes)))
        self._image_ext = '.jpg'
        self._image_index = self._load_image_set_index()

        self.proposal_file = proposal_file

        # Default to roidb handler
        if model_to_use == 'rpn':           # Step 1, 3
            self._roidb_handler = self.rpn_train_roidb
        elif model_to_use == 'frcnn':
            if proposal == 'rpn':           # Step 2, 4
                self._roidb_handler = self.rpn_proposal_roidb
            elif proposal == 'ss':
                self._roidb_handler = self.selective_search_roidb

        # PASCAL specific config options
        self.config = {'cleanup'  : True,
                       'use_salt' : True,
                       'top_k'    : 2000}

        assert os.path.exists(self._devkit_path), \
                'VOCdevkit path does not exist: {}'.format(self._devkit_path)
        assert os.path.exists(self._data_path), \
                'Path does not exist: {}'.format(self._data_path)

    def image_path_at(self, i):
        """
        Return the absolute path to image i in the image sequence.
        """
        return self.image_path_from_index(self._image_index[i])

    def image_path_from_index(self, index):
        """
        Construct an image path from the image's "index" identifier.
        """
        image_path = os.path.join(self._data_path, 'JPEGImages',
                                  index + self._image_ext)
        assert os.path.exists(image_path), \
                'Path does not exist: {}'.format(image_path)
        return image_path

    def _load_image_set_index(self):
        """
        Load the indexes listed in this dataset's image set file.
        """
        # Example path to image set file:
        # self._devkit_path + /VOCdevkit2007/VOC2007/ImageSets/Main/val.txt
        image_set_file = os.path.join(self._data_path, 'ImageSets', 'Main',
                                      self._image_set + '.txt')
        assert os.path.exists(image_set_file), \
                'Path does not exist: {}'.format(image_set_file)
        with open(image_set_file) as f:
            image_index = [x.strip() for x in f.readlines()]
        return image_index

    def gt_roidb(self):
        """
        Return the database of ground-truth regions of interest.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path, self.name + '_gt_roidb.pkl')

        # DJDJ
        if os.path.exists(cache_file):
        #if False:
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} gt roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = [self._load_pascal_annotation(index)
                    for index in self.image_index]
        with open(cache_file, 'wb') as fid:
            cPickle.dump(gt_roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote gt roidb to {}'.format(cache_file)

        return gt_roidb

    def selective_search_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_selective_search_roidb.pkl')

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()
            ss_roidb = self._load_selective_search_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        else:
            roidb = self._load_selective_search_roidb(None)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote ss roidb to {}'.format(cache_file)

        return roidb

    def _load_selective_search_roidb(self, gt_roidb):
        filename = os.path.abspath(os.path.join(self.cache_path, '..',
                                                'selective_search_data',
                                                self.name + '.mat'))
        assert os.path.exists(filename), \
               'Selective search data not found at: {}'.format(filename)
        raw_data = sio.loadmat(filename)['boxes'].ravel()

        # DJDJ
        #raw_data = raw_data[:150, ]

        box_list = []
        for i in xrange(raw_data.shape[0]):
            box_list.append(raw_data[i][:, (1, 0, 3, 2)] - 1)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def selective_search_IJCV_roidb(self):
        """
        Return the database of selective search regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        cache_file = os.path.join(self.cache_path,
                '{:s}_selective_search_IJCV_top_{:d}_roidb.pkl'.
                format(self.name, self.config['top_k']))

        if os.path.exists(cache_file):
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} ss roidb loaded from {}'.format(self.name, cache_file)
            return roidb

        gt_roidb = self.gt_roidb()
        ss_roidb = self._load_selective_search_IJCV_roidb(gt_roidb)
        roidb = datasets.imdb.merge_roidbs(gt_roidb, ss_roidb)
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote ss roidb to {}'.format(cache_file)

        return roidb

    def _load_selective_search_IJCV_roidb(self, gt_roidb):
        IJCV_path = os.path.abspath(os.path.join(self.cache_path, '..',
                                                 'selective_search_IJCV_data',
                                                 'voc_' + self._year))
        assert os.path.exists(IJCV_path), \
               'Selective search IJCV data not found at: {}'.format(IJCV_path)

        top_k = self.config['top_k']
        box_list = []
        for i in xrange(self.num_images):
            filename = os.path.join(IJCV_path, self.image_index[i] + '.mat')
            raw_data = sio.loadmat(filename)
            box_list.append((raw_data['boxes'][:top_k, :]-1).astype(np.uint16))

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def rpn_train_roidb(self):
        """
        Return the database of rpn regions of interest.
        Ground-truth ROIs are also included.
        """
        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()

            roidb = gt_roidb

        return roidb

    def rpn_proposal_roidb(self):
        """
        Return the database of rpn regions of interest.
        Ground-truth ROIs are also included.

        This function loads/saves from/to a cache file to speed up future calls.
        """
        """
        cache_file = os.path.join(self.cache_path,
                                  self.name + '_rpn_roidb.pkl')

        # DJDJ
        #if os.path.exists(cache_file):
        if False:
            with open(cache_file, 'rb') as fid:
                roidb = cPickle.load(fid)
            print '{} rpn roidb loaded from {}'.format(self.name, cache_file)
            return roidb
        """

        if int(self._year) == 2007 or self._image_set != 'test':
            gt_roidb = self.gt_roidb()

            roidb = gt_roidb

            rpn_roidb = self._load_rpn_roidb(gt_roidb)
            roidb = datasets.imdb.merge_roidbs(gt_roidb, rpn_roidb)
        else:
            roidb = self._load_rpn_roidb(None)

        """
        with open(cache_file, 'wb') as fid:
            cPickle.dump(roidb, fid, cPickle.HIGHEST_PROTOCOL)
        print 'wrote rpn roidb to {}'.format(cache_file)
        """
        return roidb

    def _load_rpn_roidb(self, gt_roidb):
        filename = os.path.abspath(self.proposal_file)
        assert os.path.exists(filename), \
               'RPN data not found at: {}'.format(filename)
        with open(filename, 'rb') as fid:
            box_list = cPickle.load(fid)

        return self.create_roidb_from_box_list(box_list, gt_roidb)

    def _load_pascal_annotation(self, index):
        """
        Load image and bounding boxes info from XML file in the PASCAL VOC
        format.
        """
        filename = os.path.join(self._data_path, 'Annotations', index + '.xml')
        # print 'Loading: {}'.format(filename)
        def get_data_from_tag(node, tag):
            return node.getElementsByTagName(tag)[0].childNodes[0].data

        with open(filename) as f:
            data = minidom.parseString(f.read())

        objs = data.getElementsByTagName('object')
        num_objs = len(objs)

        boxes = np.zeros((num_objs, 4), dtype=np.uint16)
        gt_classes = np.zeros((num_objs), dtype=np.int32)
        overlaps = np.zeros((num_objs, self.num_classes), dtype=np.float32)

        # Load object bounding boxes into a data frame.
        for ix, obj in enumerate(objs):
            # Make pixel indexes 0-based
            x1 = float(get_data_from_tag(obj, 'xmin')) - 1
            y1 = float(get_data_from_tag(obj, 'ymin')) - 1
            x2 = float(get_data_from_tag(obj, 'xmax')) - 1
            y2 = float(get_data_from_tag(obj, 'ymax')) - 1
            cls = self._class_to_ind[
                    str(get_data_from_tag(obj, "name")).lower().strip()]
            boxes[ix, :] = [x1, y1, x2, y2]
            gt_classes[ix] = cls
            overlaps[ix, cls] = 1.0

        overlaps = scipy.sparse.csr_matrix(overlaps)

        return {'boxes' : boxes,
                'gt_classes': gt_classes,
                'gt_overlaps' : overlaps,
                'label_file' : index + '.xml',
                'flipped' : False}

    def _write_voc_results_file(self, all_boxes):
        use_salt = self.config['use_salt']
        comp_id = 'comp4'
        if use_salt:
            comp_id += '-{}'.format(os.getpid())

        # VOCdevkit/results/VOC2007/Main/comp4-44503_det_test_aeroplane.txt
        path = os.path.join(self._devkit_path, 'results', 'VOC' + self._year,
                            'Main', comp_id + '_')
        for cls_ind, cls in enumerate(self.classes):
            if cls == '__background__':
                continue
            print 'Writing {} VOC results file'.format(cls)
            filename = path + 'det_' + self._image_set + '_' + cls + '.txt'
            with open(filename, 'wt') as f:
                for im_ind, index in enumerate(self.image_index):
                    dets = all_boxes[cls_ind][im_ind]
                    if dets == []:
                        continue
                    # the VOCdevkit expects 1-based indices
                    for k in xrange(dets.shape[0]):
                        f.write('{:s} {:.3f} {:.1f} {:.1f} {:.1f} {:.1f}\n'.
                                format(index, dets[k, -1],
                                       dets[k, 0] + 1, dets[k, 1] + 1,
                                       dets[k, 2] + 1, dets[k, 3] + 1))
        return comp_id

    def _do_matlab_eval(self, comp_id, output_dir='output'):
        rm_results = self.config['cleanup']

        path = os.path.join(os.path.dirname(__file__),
                            'VOCdevkit-matlab-wrapper')
        """
        cmd = 'cd {} && '.format(path)
        cmd += '{:s} -nodisplay -nodesktop '.format(datasets.MATLAB)
        cmd += '-r "dbstop if error; '
        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
               .format(self._devkit_path, comp_id,
                       self._image_set, output_dir, int(rm_results))
        """
        cmd = 'cd {} && '.format(path)
        cmd += '{:s} --eval "'.format(datasets.OCTAVE)
        cmd += 'voc_eval(\'{:s}\',\'{:s}\',\'{:s}\',\'{:s}\',{:d}); quit;"' \
               .format(self._devkit_path, comp_id,
                       self._image_set, output_dir, int(rm_results))

        print('Running:\n{}'.format(cmd))

        status = subprocess.call(cmd, shell=True)
        """
        process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

        while True:
            buff = process.stdout.readline()
            if buff == '' and process.poll() != None:
                break
            sys.stdout.write(buff)
        process.wait()
        """

    def evaluate_detections(self, all_boxes, output_dir):
        comp_id = self._write_voc_results_file(all_boxes)
        self._do_matlab_eval(comp_id, output_dir)

    def competition_mode(self, on):
        if on:
            self.config['use_salt'] = False
            self.config['cleanup'] = False
        else:
            self.config['use_salt'] = True
            self.config['cleanup'] = True

if __name__ == '__main__':
    d = datasets.pascal_voc('trainval', '2007')
    res = d.roidb
    from IPython import embed; embed()
<|endoftext|>


<commit_before>"""
Setup a custom except hook which formats exceptions that are uncaught.
In "DEBUGMODE", we'll just use the typical sys.excepthook behavior and print a
stacktrace. It's really for debugging problems with the CLI itself, but it
might also come in handy if we have issues with the way that we're trying to
format an exception.
Define an except hook per exception type that we want to treat specially,
generally types of SDK errors, and dispatch onto tht set of hooks.
"""
import sys
import os

import click

from globus_sdk import exc
from globus_cli.safeio import safeprint, write_error_info, PrintableErrorField

def pagination_overrun_hook():
    write_error_info(
        'Internal Paging Error',
        [PrintableErrorField(
            'details', ('Some kind of paging error happened. '
                        'Likely an issue with the CLI or Globus SDK.'))])

def transferapi_hook(exception):
    write_error_info(
        'Transfer API Error',
        [PrintableErrorField('HTTP status', exception.http_status),
         PrintableErrorField('request_id', exception.request_id),
         PrintableErrorField('code', exception.code),
         PrintableErrorField('message', exception.message, multiline=True)])

def globusapi_hook(exception):
    write_error_info(
        'GLobus API Error',
        [PrintableErrorField('HTTP status', exception.http_status),
         PrintableErrorField('code', exception.code),
         PrintableErrorField('message', exception.message, multiline=True)])

def custom_except_hook(exc_info):
    """
    A custom excepthook to present python errors produced by the CLI.
    We don't want to show end users big scary stacktraces if they aren't python
    programmers, so slim it down to some basic info. We keep a "DEBUGMODE" env
    variable kicking around to let us turn on stacktraces if we ever need them.

    Additionally, does global suppression of EPIPE errors, which often occur
    when a python command is piped to a consumer like `head` which closes its
    input stream before python has sent all of its output.
    DANGER: There is a (small) risk that this will bite us if there are EPIPE
    errors produced within the Globus SDK. We should keep an eye on this
    possibility, as it may demand more sophisticated handling of EPIPE.
    Possible TODO item to reduce this risk: inspect the exception and only hide
    EPIPE if it comes from within the globus_cli package.
    """
    exception_type, exception, traceback = exc_info

    # check if we're in debug mode, and run the real excepthook if we are
    if os.environ.get('GLOBUS_CLI_DEBUGMODE', None) is not None:
        sys.excepthook(exception_type, exception, traceback)

    # we're not in debug mode, do custom handling
    else:
        # if it's a click exception, re-raise as original -- Click's main
        # execution context will handle pretty-printing
        if isinstance(exception, click.ClickException):
            raise exception_type, exception, traceback

        # handle the Globus-raised errors with our special hooks
        # these will present the output (on stderr) as JSON
        elif exception_type is exc.PaginationOverrunError:
            pagination_overrun_hook()
        elif exception_type is exc.TransferAPIError:
            transferapi_hook(exception)
        elif exception_type is exc.GlobusAPIError:
            globusapi_hook(exception)
        else:
            safeprint('{}: {}'.format(exception_type.__name__, exception))
<commit_msg>Expand JSON error handling to cover non-API errors<commit_after>"""
Setup a custom except hook which formats exceptions that are uncaught.
In "DEBUGMODE", we'll just use the typical sys.excepthook behavior and print a
stacktrace. It's really for debugging problems with the CLI itself, but it
might also come in handy if we have issues with the way that we're trying to
format an exception.
Define an except hook per exception type that we want to treat specially,
generally types of SDK errors, and dispatch onto tht set of hooks.
"""
import sys
import os

import click

from globus_sdk import exc
from globus_cli.safeio import safeprint, write_error_info, PrintableErrorField

def pagination_overrun_hook():
    write_error_info(
        'Internal Paging Error',
        [PrintableErrorField(
            'details', ('Some kind of paging error happened. '
                        'Likely an issue with the CLI or Globus SDK.'))])

def transferapi_hook(exception):
    write_error_info(
        'Transfer API Error',
        [PrintableErrorField('HTTP status', exception.http_status),
         PrintableErrorField('request_id', exception.request_id),
         PrintableErrorField('code', exception.code),
         PrintableErrorField('message', exception.message, multiline=True)])

def globusapi_hook(exception):
    write_error_info(
        'GLobus API Error',
        [PrintableErrorField('HTTP status', exception.http_status),
         PrintableErrorField('code', exception.code),
         PrintableErrorField('message', exception.message, multiline=True)])

def globus_generic_hook(exception):
    write_error_info(
        'Globus Error',
        [PrintableErrorField('error_type', exception.__class__.__name__),
         PrintableErrorField('message', str(exception), multiline=True)])

def custom_except_hook(exc_info):
    """
    A custom excepthook to present python errors produced by the CLI.
    We don't want to show end users big scary stacktraces if they aren't python
    programmers, so slim it down to some basic info. We keep a "DEBUGMODE" env
    variable kicking around to let us turn on stacktraces if we ever need them.

    Additionally, does global suppression of EPIPE errors, which often occur
    when a python command is piped to a consumer like `head` which closes its
    input stream before python has sent all of its output.
    DANGER: There is a (small) risk that this will bite us if there are EPIPE
    errors produced within the Globus SDK. We should keep an eye on this
    possibility, as it may demand more sophisticated handling of EPIPE.
    Possible TODO item to reduce this risk: inspect the exception and only hide
    EPIPE if it comes from within the globus_cli package.
    """
    exception_type, exception, traceback = exc_info

    # check if we're in debug mode, and run the real excepthook if we are
    if os.environ.get('GLOBUS_CLI_DEBUGMODE', None) is not None:
        sys.excepthook(exception_type, exception, traceback)

    # we're not in debug mode, do custom handling
    else:
        # if it's a click exception, re-raise as original -- Click's main
        # execution context will handle pretty-printing
        if isinstance(exception, click.ClickException):
            raise exception_type, exception, traceback

        # handle the Globus-raised errors with our special hooks
        # these will present the output (on stderr) as JSON
        elif exception_type is exc.PaginationOverrunError:
            pagination_overrun_hook()
        elif exception_type is exc.TransferAPIError:
            transferapi_hook(exception)
        elif exception_type is exc.GlobusAPIError:
            globusapi_hook(exception)

        # specific checks fell through -- now check if it's any kind of
        # GlobusError
        elif isinstance(exception, exc.GlobusError):
            globus_generic_hook(exception)

        # not a GlobusError, not a ClickException -- something like ValueError
        # or NotImplementedError bubbled all the way up here: just print it
        # out, basically
        else:
            safeprint('{}: {}'.format(exception_type.__name__, exception))
<|endoftext|>


<commit_before>#!/usr/bin/env python3

"""
Reads one or more (optionally gz compressed) FASTA files and reports some basic statistics.  Example:

$ ./fasta_simple_stats.py my.fasta

File: my.fastq
     Total sequence entries: 277971801
     Total bases: 28353123702
     Avg sequence length: 102.0
     GC percentage: 44.5

If you pass multiple files via *.fasta or something similar this will report stats
on these as an aggregate unless you pass the --individual flag, which will cause
reporting on each file individually.

Contact: jorvis@gmail.com
"""

import argparse
import codecs
import gzip
import os
import sys

def main():
    parser = argparse.ArgumentParser( description='Provides simple quantitative statistics for one or more FASTA files')

    ## output file to be written
    parser.add_argument('input_files', metavar='N', type=str, nargs='+', help='Path to one or more input files, separated by spaces' )
    parser.add_argument('-o', '--output_file', type=str, required=False, help='Optional path to an output file to be created, else prints on STDOUT' )
    parser.add_argument('--individual', action='store_true', help='Report stats on each file individually' )
    args = parser.parse_args()

    ## open the output file
    fout = None
    if args.output_file is None:
        fout = codecs.getwriter('utf8')(sys.stdout.buffer)
    else:
        fout = open(args.output_file, "w")

    ## values that will be reported
    entry_count = 0
    total_bases = 0
    gc_count    = 0

    for input_file in args.input_files:
        line_number = 0

        if input_file.endswith('.gz'):
            fh = gzip.open(input_file, 'rb')
            is_compressed = True
        else:
            fh = open(input_file, 'r')
            is_compressed = False

        if args.individual:
            entry_count = 0
            total_bases = 0
            gc_count    = 0

        for line in fh:
            if is_compressed:
                line = line.decode()

            line = line.rstrip()

            ## every line starting with > is the start of a sequence entry
            if line[0] == '>':
                entry_count += 1
            else:
                total_bases += len(line)
                gc_count += line.count('G')
                gc_count += line.count('C')

        if args.individual:
            report_stats(fout, input_file, entry_count, total_bases, gc_count)

    if not args.individual:
        if len(args.input_files) == 1:
            report_stats(fout, args.input_files[0], entry_count, total_bases, gc_count)
        else:
            report_stats(fout, 'multiple', entry_count, total_bases, gc_count)

def report_stats(fout, input_file, entry_count, total_bases, gc_count):
    avg_entry_length = total_bases / entry_count

    fout.write("File: {0}\n".format(input_file))
    fout.write("\tTotal sequence entries: {0}\n".format(entry_count))
    fout.write("\tTotal bases: {0}\n".format(total_bases))
    fout.write("\tAvg sequence length: {0:.1f}\n".format(avg_entry_length))
    fout.write("\tGC percentage: {0:.1f}\n".format( (gc_count / total_bases) * 100))

if __name__ == '__main__':
    main()

<commit_msg>Added shortest/longest sequence to output, some format changes<commit_after>#!/usr/bin/env python3

"""
Reads one or more (optionally gz compressed) FASTA files and reports some basic statistics.  Example:

$ ./fasta_simple_stats.py Trinity.fasta

File: Trinity.fasta
	Total sequence entries: 583596
	Total bases: 425253191
	Shortest sequence length: 177.0 (TRINITY_DN32084_c0_g1_i3)
	Average sequence length : 728.7
	Longest sequence length : 42580.0 (TRINITY_DN1050_c0_g1_i9)
	GC percentage: 43.2

If you pass multiple files via *.fasta or something similar this will report stats
on these as an aggregate unless you pass the --individual flag, which will cause
reporting on each file individually.

Contact: jorvis@gmail.com
"""

import argparse
import codecs
import gzip
import os
import sys
import re

def main():
    parser = argparse.ArgumentParser( description='Provides simple quantitative statistics for one or more FASTA files')

    ## output file to be written
    parser.add_argument('input_files', metavar='N', type=str, nargs='+', help='Path to one or more input files, separated by spaces' )
    parser.add_argument('-o', '--output_file', type=str, required=False, help='Optional path to an output file to be created, else prints on STDOUT' )
    parser.add_argument('--individual', action='store_true', help='Report stats on each file individually' )
    args = parser.parse_args()

    ## open the output file
    fout = None
    if args.output_file is None:
        fout = codecs.getwriter('utf8')(sys.stdout.buffer)
    else:
        fout = open(args.output_file, "w")

    ## values that will be reported
    data = {
        'entry_count': 0,
        'total_bases': 0,
        'gc_count': 0,
        'smallest_length': None,
        'smallest_id': None,
        'largest_length': 0,
        'largest_id': None
    }

    for input_file in args.input_files:
        if input_file.endswith('.gz'):
            fh = gzip.open(input_file, 'rb')
            is_compressed = True
        else:
            fh = open(input_file, 'r')
            is_compressed = False

        if args.individual:
           data = {
               'entry_count': 0,
               'total_bases': 0,
               'gc_count': 0,
               'smallest_length': None,
               'smallest_id': None,
               'largest_length': 0,
               'largest_id': None
           }

        line_number = 0
        this_seq_length = 0

        for line in fh:
            line_number += 1
            if is_compressed:
                line = line.decode()

            line = line.rstrip()

            ## every line starting with > is the start of a sequence entry
            if line[0] == '>':
                data['entry_count'] += 1

                if line_number != 1:
                    if data['smallest_id'] is None:
                        # this is the first record, so it sets all records.
                        data['smallest_id'] = seq_id
                        data['smallest_length'] = this_seq_length
                    elif this_seq_length < data['smallest_length']:
                        data['smallest_id'] = seq_id
                        data['smallest_length'] = this_seq_length

                    if this_seq_length > data['largest_length']:
                        data['largest_length'] = this_seq_length
                        data['largest_id'] = seq_id

                m = re.match(">(\S+)", line)
                seq_id = m.group(1)

                this_seq_length = 0
            else:
                data['total_bases'] += len(line)
                this_seq_length += len(line)
                data['gc_count'] += line.count('G')
                data['gc_count'] += line.count('C')

        if args.individual:
            report_stats(fout, input_file, data)

    if not args.individual:
        if len(args.input_files) == 1:
            report_stats(fout, args.input_files[0], data)
        else:
            report_stats(fout, 'multiple', data)

def report_stats(fout, input_file, data):
    avg_entry_length = data['total_bases'] / data['entry_count']

    fout.write("File: {0}\n".format(input_file))
    fout.write("\tTotal sequence entries: {0}\n".format(data['entry_count']))
    fout.write("\tTotal bases: {0}\n".format(data['total_bases']))
    fout.write("\tShortest sequence length: {0:.1f} ({1})\n".format(data['smallest_length'], data['smallest_id']))
    fout.write("\tAverage sequence length : {0:.1f}\n".format(avg_entry_length))
    fout.write("\tLongest sequence length : {0:.1f} ({1})\n".format(data['largest_length'], data['largest_id']))
    fout.write("\tGC percentage: {0:.1f}\n".format( (data['gc_count'] / data['total_bases']) * 100))

if __name__ == '__main__':
    main()

<|endoftext|>


<commit_before>from django.test import TestCase

# Create your tests here.
<commit_msg>Testing fieldsettings.<commit_after>import datetime
from django.test import TestCase
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.core.exceptions import ValidationError
from feds.settings import FEDS_BASIC_SETTING_GROUP, FEDS_INTEGER_SETTING
from .models import FieldSetting

class FedsFieldSettingsTests(TestCase):
    def setUp(self):
        # Make some users to be project owners.
        # self.u1 = User.objects.create_user('u1', 'u1@example.com', 'u1')
        # self.u2 = User.objects.create_user('u2', 'u2@example.com', 'u2')
        pass

    def test_make_setting_ok(self):
        t = "Dogs!"
        ba = FieldSetting()
        ba.title = t
        ba.setting_group = FEDS_BASIC_SETTING_GROUP
        ba.setting_type = FEDS_INTEGER_SETTING
        ba.setting_params = '{"title": "Dogs are the best!"}'
        ba.save()

        ba2 = FieldSetting.objects.get(pk=ba.pk)
        self.assertEqual(ba2.title, t)

    def test_make_setting_no_title(self):
        with self.assertRaises(ValidationError):
            ba = FieldSetting()
            ba.setting_group = FEDS_BASIC_SETTING_GROUP
            ba.setting_type = FEDS_INTEGER_SETTING
            ba.setting_params = '{"title": "Dogs are the best!"}'
            ba.save()

    def test_make_setting_whitespace_title(self):
        with self.assertRaises(ValidationError):
            ba = FieldSetting()
            ba.title = '   '
            ba.setting_group = FEDS_BASIC_SETTING_GROUP
            ba.setting_type = FEDS_INTEGER_SETTING
            ba.setting_params = '{"title": "Dogs are the best!"}'
            ba.save()

    def test_make_setting_bad_group(self):
        with self.assertRaises(ValidationError):
            ba = FieldSetting()
            ba.title = 'DOGS!'
            ba.setting_group = 'The Beatles'
            ba.setting_type = FEDS_INTEGER_SETTING
            ba.setting_params = '{"title": "Dogs are the best!"}'
            ba.save()

    def test_make_setting_bad_type(self):
        with self.assertRaises(ValidationError):
            ba = FieldSetting()
            ba.title = 'Rosie!'
            ba.setting_group = FEDS_BASIC_SETTING_GROUP
            ba.setting_type = 'Bad'
            ba.setting_params = '{"title": "Dogs are the best!"}'
            ba.save()

    def test_make_setting_bad_params(self):
        with self.assertRaises(ValidationError):
            ba = FieldSetting()
            ba.title = 'Rosie!'
            ba.setting_group = FEDS_BASIC_SETTING_GROUP
            ba.setting_type = FEDS_INTEGER_SETTING
            ba.setting_params = '{title": "Dogs are the best!"}'
            ba.save()

<|endoftext|>


<commit_before>#!/usr/bin/env python

import sys, os
import codecs
import datetime

import mrjob.job
import mrjob.protocol

try: # workaround
    import util
    from base_etl_job import BaseETLJob
except ImportError:
    pass

class SearchCountJob(BaseETLJob):

    def mapper(self, _, line):

        sep = self.options.field_separator
        snapshot_date = self.options.snapshot_date

        def make_key(envx, used_on_dt, srch_loc, srch_partner):
            try:
                d = datetime.datetime.strptime(used_on_dt, "%Y-%m-%d")
                if not util.is_valid_date(d):
                    raise ValueError
            except ValueError:
                self.mark_invalid_date()

            key_fields = [ snapshot_date,
                           '3',
                           d.strftime("%Y-%m-%d"),
                           envx["product"],
                           envx["version"],
                           envx["channel"],
                           envx.get("locale", '#unknown'),
                           envx["country"] ] + \
                           envx["os"] + \
                           [ srch_partner,
                             srch_loc ]

            return util.strip_invalid_chars(sep.join(key_fields))

        rec = self.get_fhr_report(line)
        if not rec:
            return

        try:
            data_days = rec.getDaysData()
            pkn = 'org.mozilla.searches.counts'

            for dobj in data_days:
                for location, counts in dobj.data.get(pkn, {}).iteritems():
                    if not isinstance(counts, dict):
                        continue
                    for partner, cnt in counts.iteritems():
                        try:
                            yield make_key(dobj.env, dobj.date, location, partner), (1, cnt)
                        except ValueError:
                            pass

        except Exception as e:
            self.mark_invalid_report()

    def combiner(self, key, vals):
        n, c = map(sum, apply(zip, list(vals)))
        yield key, (n ,c)

    def reducer(self, key, vals):
        n, c = map(sum, apply(zip, list(vals)))
        yield None, self.options.field_separator.join([key, str(n), str(c)])

if __name__ == '__main__':
    SearchCountJob.run()

<commit_msg>Fixes bug (incorrect key separator) in search_counts job<commit_after>#!/usr/bin/env python

import sys, os
import codecs
import datetime

import mrjob.job
import mrjob.protocol

try: # workaround
    import util
    from base_etl_job import BaseETLJob
except ImportError:
    pass

class SearchCountJob(BaseETLJob):

    def mapper(self, _, line):

        sep = self.options.field_separator
        snapshot_date = self.options.snapshot_date

        def make_key(envx, used_on_dt, srch_loc, srch_partner):
            try:
                d = datetime.datetime.strptime(used_on_dt, "%Y-%m-%d")
                if not util.is_valid_date(d):
                    raise ValueError
            except ValueError:
                self.mark_invalid_date()

            key_fields = [ snapshot_date,
                           '3',
                           d.strftime("%Y-%m-%d"),
                           envx["product"],
                           envx["version"],
                           envx["channel"],
                           envx.get("locale", '#unknown'),
                           envx["country"] ] + \
                           envx["os"] + \
                           [ srch_partner,
                             srch_loc ]

            return sep.join(map(util.strip_invalid_chars, key_fields))

        rec = self.get_fhr_report(line)
        if not rec:
            return

        try:
            data_days = rec.getDaysData()
            pkn = 'org.mozilla.searches.counts'

            for dobj in data_days:
                for location, counts in dobj.data.get(pkn, {}).iteritems():
                    if not isinstance(counts, dict):
                        continue
                    for partner, cnt in counts.iteritems():
                        try:
                            yield make_key(dobj.env, dobj.date, location, partner), (1, cnt)
                        except ValueError:
                            pass

        except Exception as e:
            self.mark_invalid_report()

    def combiner(self, key, vals):
        n, c = map(sum, apply(zip, list(vals)))
        yield key, (n ,c)

    def reducer(self, key, vals):
        n, c = map(sum, apply(zip, list(vals)))
        yield None, self.options.field_separator.join([key, str(n), str(c)])

if __name__ == '__main__':
    SearchCountJob.run()

<|endoftext|>


<commit_before>#test_patchboard.py
#
# Copyright 2014 BitVault.

from __future__ import print_function

import json
import pytest

from patchboard.tests.fixtures import pb
pytest.mark.usefixtures(pb)

def test_endpoint_classes(pb):
    with open(u"patchboard/tests/data/endpoint_classes.json", u"r") as file:
        ruby_classes = json.load(file)

    python_classes = pb.endpoint_classes

    assert len(ruby_classes) == len(python_classes)

    ruby_keys = ruby_classes.keys()
    ruby_keys.sort()

    python_keys = python_classes.keys()
    python_keys.sort()

    assert ruby_keys == python_keys

    for key in python_keys:
        assert (
            ruby_classes[key].split(u'::')[-1] ==
            python_classes[key].__name__)

# This doesn't work well, as too many automatic methods get mixed
# in in both languages.
# FIXME: this will also pick up data attributes on the python side.
def test_endpoint_object_methods(pb):
    with open(
            u"patchboard/tests/data/endpoint_object_methods.json", u"r") as file:
        ruby_object_methods = json.load(file)

    for key in ruby_object_methods.keys():
        ruby_object_methods[key] = [
            method for method in ruby_object_methods[key]
            if method not in ("method_missing")]
        ruby_object_methods[key].sort

    python_object_methods = {}
    for clsname, cls in pb.endpoint_classes.iteritems():
        # Use dict so we don't see inherited methods
        method_list = cls.__dict__.keys()
        method_list = [method for method in method_list
                       if method[0] != '_']
        method_list.sort()
        python_object_methods[clsname] = method_list

    assert ruby_object_methods == python_object_methods
<commit_msg>Fix test<commit_after>#test_patchboard.py
#
# Copyright 2014 BitVault.

from __future__ import print_function

import json
import pytest

from patchboard.tests.fixtures import pb
pytest.mark.usefixtures(pb)

def test_endpoint_classes(pb):
    with open(u"patchboard/tests/data/endpoint_classes.json", u"r") as file:
        ruby_classes = json.load(file)

    python_classes = pb.endpoint_classes

    assert len(ruby_classes) == len(python_classes)

    ruby_keys = ruby_classes.keys()
    ruby_keys.sort()

    python_keys = python_classes.keys()
    python_keys.sort()

    assert ruby_keys == python_keys

    for key in python_keys:
        assert (
            ruby_classes[key].split(u'::')[-1] ==
            python_classes[key].__name__)

# This doesn't work well, as too many automatic methods get mixed
# in in both languages.
# FIXME: this will also pick up data attributes on the python side.
def test_endpoint_object_methods(pb):
    with open(
            u"patchboard/tests/data/endpoint_object_methods.json", u"r") as file:
        ruby_object_methods = json.load(file)

    for key in ruby_object_methods.keys():
        ruby_object_methods[key] = sorted(ruby_object_methods[key])

    python_object_methods = {}
    for clsname, cls in pb.endpoint_classes.iteritems():
        # Use dict so we don't see inherited methods
        method_list = cls.__dict__.keys()
        method_list = [method for method in method_list
                       if method[0] != '_']
        python_object_methods[clsname] = sorted(method_list)

    assert ruby_object_methods == python_object_methods
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
from vhdl_objects.entity import Entity
from vhdl_objects.wire import Wire
from vhdl_objects.library import Library

class Vhdl_reader:
    """
    Vhdl_reader take the .vhd file and return a full entity
    """
    def __init__(self, filename):
        self.state = "start_parsing"
        self.lib_part = ""
        self.entity_generic_part = ""
        self.entity_port_part = ""
        self.entity_part = ""

        self.long_file_name = filename
        self.extract_file_name(self.long_file_name)

        self.file = None
        self.entity = None
        self.entity_bloc = ""

        self.open_file()
        self.entity = Entity()
        self.parse_vhdl_file()
        self.parse_entity_part()
        self.verbose()
        self.close_file()

    def extract_file_name(self, long_file_name):
        self.filename = long_file_name.split("/")[-1]

    def parse_vhdl_file(self):
        for raw_line in self.file:
            raw_line = self.clean_line(raw_line)
            real_words = raw_line.split()
            # remove comment
            clean_words = raw_line.lower().split()

            # remove blank line
            if len(clean_words) == 0:
                continue

            if self.state == "start_parsing":
                if "entity" in clean_words:
                    if "is" in clean_words:
                        the_index = clean_words.index("entity")
                        self.entity.set_name(real_words[the_index + 1])
                        self.state = "parse_entity"
                        if "port" in clean_words:
                            self.entity_part += "port"
                            if "(" in clean_words:
                                self.entity_part += "(\n"
                            else:
                                self.entity_part += "\n"
                else:
                    self.lib_part += self.remove_comment(raw_line)
            else:
                if self.state == "parse_entity":
                    if "end" in clean_words:
                        the_index = clean_words.index("end")

                        if real_words[the_index + 1] == self.entity.name or\
                                real_words[the_index + 1] == self.entity.name + ";":
                            self.state = "after_entity"
                    else:
                        self.entity_part += self.remove_comment(raw_line)
                else:
                    if self.state == "after_entity":
                        # To continue
                        break

    def parse_entity_part(self):
        state = "start_parsing"
        for raw_line in self.entity_part.split(";"):
            print raw_line
            raw_line = self.clean_line(raw_line)
            clean_words = raw_line.lower().split()
            # Generic not used so we neglect it
            if state == "start_parsing":
                if "port" in clean_words:
                    state = "parse_port"
                    print raw_line
                    raw_line = self.remove_port_from_text(raw_line)
                    print raw_line

            if state == "parse_port":
                clean_words = self.clean_line(raw_line).split()
                if len(clean_words) == 0:
                    continue
                self.extract_wire(raw_line)

        pass

    def remove_port_from_text(self, text):
        words_lower = text.split()
        index = words_lower.index("(")
        words = words_lower[index+1:]
        print words
        return " ".join(words)

    def extract_wire(self, text):
        if "clk" in text.lower():
            print text.lower()
            wire_property = "clk"
        else:
            wire_property = "classic"

        real_words = text.split()
        wire_type = real_words[3].lower()
        if wire_type == "integer" or\
                wire_type == "natural" or\
                wire_type == "positive":

            nb_wires = 32

        if wire_type == "std_logic":
            nb_wires = 1

        if wire_type == "std_logic_vector" or\
                wire_type == "unsigned" or\
                wire_type == "signed" :

            bus_direction = real_words[6].lower()
            bus_description = real_words[5:8]
            if bus_direction == "downto":
                upper_val = bus_description[0]
                lower_val = bus_description[2]
                try:  # if upper_val and lower_val are integers
                    nb_wires = int(upper_val) - int(lower_val) + 1
                except:
                    nb_wires = self.compute_wire_number(
                            upper_val, lower_val)

            else:
                upper_val = bus_description[2]
                lower_val = bus_description[0]
                try:  # if upper_val and lower_val are integers
                    nb_wires = int(upper_val) - int(lower_val) + 1
                except:
                    nb_wires = self.compute_wire_number(
                    upper_val, lower_val)

        if real_words[2] == "in":
            self.entity.add_input(Wire(real_words[0], nb_wires, wire_property))

        if real_words[2] == "out" or real_words[2] == "buffer":
            self.entity.add_output(Wire(real_words[0], nb_wires, wire_property))

        if real_words[2] == "inout":
            self.entity.add_inout(Wire(real_words[0], nb_wires, wire_property))

    def compute_wire_number(self, up, low):
        low = low.replace(" ", "")
        try:
            upper_val = int(up)
            return self.wire_number_upper_is_int(upper_val, low)
        except:
            left_up = up.split("-")[0]
            right_up = up.split("-")[1]
            lower_val = int(low)
            return self.wire_number_upper_is_not_int(left_up, right_up, lower_val)

    def wire_number_upper_is_not_int(self, left_up, right_up, lower_val):
        if -int(right_up) - lower_val + 1 == 0:
            return left_up
        else:
            return left_up + "-" + "%s" % -(-int(right_up) - lower_val + 1)

    def wire_number_upper_is_int(self, upper_val, low):
        try:
            lower_val = int(low)
            return int(upper_val) - int(lower_val) + 1
        except:
            left_low = low.split("-")[0]
            right_low = low.split("-")[1]
            return - int(right_low) + 1 + "-" + upper_val

    def clean_line(self, text):
        clean_line = self.remove_comment(text)
        clean_line = clean_line.replace(':', ' : ')
        clean_line = clean_line.replace('(', ' ( ')
        clean_line = clean_line.replace(')', ' ) ')
        clean_line = clean_line.replace(' -', '-')
        clean_line = clean_line.replace('- ', '-')
        clean_line = clean_line.replace(' ;', ';')
        return clean_line

    def remove_comment(self, text):
        words = text.split()
        clean_line = ""
        for i in range(0, len(words)):
            if words[i] == "--":
                break
            clean_line += words[i] + " "
        return clean_line

    def verbose(self):
        print "input file: %s" % self.filename
        print "entity name: %s" % self.entity.name
        print "%d inputs" % len(self.entity.inputs)
        for i in range(0, len(self.entity.inputs)):
            self.entity.inputs[i].verbose()

        print "%d outputs" % len(self.entity.outputs)
        for i in range(0, len(self.entity.outputs)):
            self.entity.outputs[i].verbose()

        print "%d inouts" % len(self.entity.inouts)
        for i in range(0, len(self.entity.inouts)):
            self.entity.inouts[i].verbose()

        pass

    def open_file(self):
        self.file = open(self.long_file_name, "r")
        pass

    def close_file(self):
        self.file.close()
        pass
<commit_msg>debug sentences removed + pep8 validation<commit_after>#!/usr/bin/python
# -*- coding: utf-8 -*-

import os
from vhdl_objects.entity import Entity
from vhdl_objects.wire import Wire
from vhdl_objects.library import Library

class Vhdl_reader:

    """
    Vhdl_reader take the .vhd file and return a full entity
    """

    def __init__(self, filename):
        self.state = "start_parsing"
        self.lib_part = ""
        self.entity_generic_part = ""
        self.entity_port_part = ""
        self.entity_part = ""

        self.long_file_name = filename
        self.extract_file_name(self.long_file_name)

        self.file = None
        self.entity = None
        self.entity_bloc = ""

        self.open_file()
        self.entity = Entity()
        self.parse_vhdl_file()
        self.parse_entity_part()
        self.verbose()
        self.close_file()

    def extract_file_name(self, long_file_name):
        self.filename = long_file_name.split("/")[-1]

    def parse_vhdl_file(self):
        for raw_line in self.file:
            raw_line = self.clean_line(raw_line)
            real_words = raw_line.split()
            # remove comment
            clean_words = raw_line.lower().split()

            # remove blank line
            if len(clean_words) == 0:
                continue

            if self.state == "start_parsing":
                if "entity" in clean_words:
                    if "is" in clean_words:
                        the_index = clean_words.index("entity")
                        self.entity.set_name(real_words[the_index + 1])
                        self.state = "parse_entity"
                        if "port" in clean_words:
                            self.entity_part += "port"
                            if "(" in clean_words:
                                self.entity_part += "(\n"
                            else:
                                self.entity_part += "\n"
                else:
                    self.lib_part += self.remove_comment(raw_line)
            else:
                if self.state == "parse_entity":
                    if "end" in clean_words:
                        the_index = clean_words.index("end")

                        if real_words[the_index + 1] == self.entity.name or\
                                real_words[the_index + 1] == self.entity.name + ";":
                            self.state = "after_entity"
                    else:
                        self.entity_part += self.remove_comment(raw_line)
                else:
                    if self.state == "after_entity":
                        # To continue
                        break

    def parse_entity_part(self):
        state = "start_parsing"
        for raw_line in self.entity_part.split(";"):
            raw_line = self.clean_line(raw_line)
            clean_words = raw_line.lower().split()
            # Generic not used so we neglect it
            if state == "start_parsing":
                if "port" in clean_words:
                    state = "parse_port"
                    raw_line = self.remove_port_from_text(raw_line)

            if state == "parse_port":
                clean_words = self.clean_line(raw_line).split()
                if len(clean_words) == 0:
                    continue
                self.extract_wire(raw_line)

        pass

    def remove_port_from_text(self, text):
        words_lower = text.split()
        index = words_lower.index("(")
        words = words_lower[index + 1:]
        return " ".join(words)

    def extract_wire(self, text):
        if "clk" in text.lower():
            wire_property = "clk"
        else:
            wire_property = "classic"

        real_words = text.split()
        wire_type = real_words[3].lower()
        if wire_type == "integer" or\
                wire_type == "natural" or\
                wire_type == "positive":

            nb_wires = 32

        if wire_type == "std_logic":
            nb_wires = 1

        if wire_type == "std_logic_vector" or\
                wire_type == "unsigned" or\
                wire_type == "signed":

            bus_direction = real_words[6].lower()
            bus_description = real_words[5:8]
            if bus_direction == "downto":
                upper_val = bus_description[0]
                lower_val = bus_description[2]
                try:  # if upper_val and lower_val are integers
                    nb_wires = int(upper_val) - int(lower_val) + 1
                except:
                    nb_wires = self.compute_wire_number(
                        upper_val, lower_val)

            else:
                upper_val = bus_description[2]
                lower_val = bus_description[0]
                try:  # if upper_val and lower_val are integers
                    nb_wires = int(upper_val) - int(lower_val) + 1
                except:
                    nb_wires = self.compute_wire_number(
                        upper_val, lower_val)

        if real_words[2] == "in":
            self.entity.add_input(Wire(real_words[0], nb_wires, wire_property))

        if real_words[2] == "out" or real_words[2] == "buffer":
            self.entity.add_output(
                Wire(real_words[0], nb_wires, wire_property))

        if real_words[2] == "inout":
            self.entity.add_inout(Wire(real_words[0], nb_wires, wire_property))

    def compute_wire_number(self, up, low):
        low = low.replace(" ", "")
        try:
            upper_val = int(up)
            return self.wire_number_upper_is_int(upper_val, low)
        except:
            left_up = up.split("-")[0]
            right_up = up.split("-")[1]
            lower_val = int(low)
            return self.wire_number_upper_is_not_int(left_up, right_up, lower_val)

    def wire_number_upper_is_not_int(self, left_up, right_up, lower_val):
        if -int(right_up) - lower_val + 1 == 0:
            return left_up
        else:
            return left_up + "-" + "%s" % -(-int(right_up) - lower_val + 1)

    def wire_number_upper_is_int(self, upper_val, low):
        try:
            lower_val = int(low)
            return int(upper_val) - int(lower_val) + 1
        except:
            left_low = low.split("-")[0]
            right_low = low.split("-")[1]
            return - int(right_low) + 1 + "-" + upper_val

    def clean_line(self, text):
        clean_line = self.remove_comment(text)
        clean_line = clean_line.replace(':', ' : ')
        clean_line = clean_line.replace('(', ' ( ')
        clean_line = clean_line.replace(')', ' ) ')
        clean_line = clean_line.replace(' -', '-')
        clean_line = clean_line.replace('- ', '-')
        clean_line = clean_line.replace(' ;', ';')
        return clean_line

    def remove_comment(self, text):
        words = text.split()
        clean_line = ""
        for i in range(0, len(words)):
            if words[i] == "--":
                break
            clean_line += words[i] + " "
        return clean_line

    def verbose(self):
        print "input file: %s" % self.filename
        print "entity name: %s" % self.entity.name
        print "%d inputs" % len(self.entity.inputs)
        for i in range(0, len(self.entity.inputs)):
            self.entity.inputs[i].verbose()

        print "%d outputs" % len(self.entity.outputs)
        for i in range(0, len(self.entity.outputs)):
            self.entity.outputs[i].verbose()

        print "%d inouts" % len(self.entity.inouts)
        for i in range(0, len(self.entity.inouts)):
            self.entity.inouts[i].verbose()

        pass

    def open_file(self):
        self.file = open(self.long_file_name, "r")
        pass

    def close_file(self):
        self.file.close()
        pass
<|endoftext|>


<commit_before># encoding: utf-8

import re

import click
from prettytable import PrettyTable

import googleanalytics as ga
from .common import cli

def table(rows, keys):
    t = PrettyTable(keys)
    t.align = 'l'
    for row in rows:
        t.add_row(ga.utils.pick(row, keys))
    return t

# TODO: after playing around with this, it seems like a recursive view
# would actually be more useful, so you can see accounts, webproperties
# and profiles all at the same time (perhaps configurable with --depth)
@cli.command()
@click.pass_obj
def properties(scope):
    if isinstance(scope, ga.account.WebProperty):
        click.echo(table(scope.profiles, ['name', 'id']))
    elif isinstance(scope, ga.account.Account):
        click.echo(table(scope.webproperties, ['name', 'url', 'id']))
    else:
        click.echo(table(scope, ['name', 'id']))

def matcher(pattern):
    def match(column):
        return re.search(pattern, column.name, re.IGNORECASE)
    return match

@cli.command()
@click.argument('pattern', required=False)
@click.pass_obj
def columns(scope, pattern=None, column_type='columns'):
    if not isinstance(scope, ga.account.Profile):
        raise ValueError('Please specify an account and webproperty.')

    columns = getattr(scope.core, column_type)

    if pattern:
        columns = filter(matcher(pattern), columns)

    click.echo(table(columns, ['name', 'slug']))
<commit_msg>Feature: add --realtime flag to columns command.<commit_after># encoding: utf-8

import re

import click
from prettytable import PrettyTable

import googleanalytics as ga
from .common import cli

def table(rows, keys):
    t = PrettyTable(keys)
    t.align = 'l'
    for row in rows:
        t.add_row(ga.utils.pick(row, keys))
    return t

# TODO: after playing around with this, it seems like a recursive view
# would actually be more useful, so you can see accounts, webproperties
# and profiles all at the same time (perhaps configurable with --depth)
@cli.command()
@click.pass_obj
def properties(scope):
    if isinstance(scope, ga.account.WebProperty):
        click.echo(table(scope.profiles, ['name', 'id']))
    elif isinstance(scope, ga.account.Account):
        click.echo(table(scope.webproperties, ['name', 'url', 'id']))
    else:
        click.echo(table(scope, ['name', 'id']))

def matcher(pattern):
    def match(column):
        return re.search(pattern, column.name, re.IGNORECASE)
    return match

@cli.command()
@click.argument('pattern', required=False)
@click.option('--realtime',
    is_flag=True,
    help='Use the RealTime API instead of the Core API.')
@click.pass_obj
def columns(scope, pattern=None, realtime=False, column_type='columns'):
    if not isinstance(scope, ga.account.Profile):
        raise ValueError('Please specify an account and webproperty.')

    if realtime:
        api = scope.realtime
    else:
        api = scope.core

    columns = getattr(api, column_type)

    if pattern:
        columns = filter(matcher(pattern), columns)

    click.echo(table(columns, ['name', 'slug']))
<|endoftext|>


<commit_before>import asyncio
import logging
import time
from concurrent.futures import CancelledError
from datetime import datetime
from threading import Thread

import pynvml
from toolz.functoolz import compose

logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG)

def nativestr(s):
    if isinstance(s, str):
        return s
    return s.decode('utf-8', 'replace')

def device_name_for(device_handle):
    """Get GPU device name"""
    try:
        return nativestr(pynvml.nvmlDeviceGetName(device_handle))
    except pynvml.NVMlError:
        return "NVIDIA"

def mem_for(device_handle):
    """Get GPU device memory consumption in percent"""
    try:
        memory_info = pynvml.nvmlDeviceGetMemoryInfo(device_handle)
        return memory_info.used * 100.0 / memory_info.total
    except pynvml.NVMLError:
        return None

def utilization_for(device_handle):
    """Get GPU device consumption in percent"""
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).gpu
    except pynvml.NVMLError:
        return None

def mem_utilization_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).memory
    except pynvml.NVMLError:
        return None

def power_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetPowerUsage(device_handle)
    except pynvml.NVMLError:
        return None

def temperature_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetTemperature(device_handle, 0)
    except pynvml.NVMLError:
        return None

_MEASUREMENTS_FUNCS ={
    "Name": device_name_for,
    "Memory": mem_for,
    "Utilization": utilization_for,
    "Memory Utilization": mem_utilization_for,
    "Power": power_for,
    "Temperature": temperature_for,
}

def measurements_for(gpu_handle):
    mes_dict = {k: func(gpu_handle) for k, func in _MEASUREMENTS_FUNCS.items()}
    mes_dict['timestamp'] = str(datetime.now())
    return mes_dict

def aggregate_measurements(device_count):
    measures_for_device = compose(measurements_for,
                                  pynvml.nvmlDeviceGetHandleByIndex)
    return {i:measures_for_device(i) for i in range(device_count)}

async def record_measurements(async_recording_func, deviceCount, polling_interval=1):
    try:
        while True:
            await async_recording_func(deviceCount)
            await asyncio.sleep(polling_interval)
    except CancelledError:  # TODO: Better control for aync loop
        print("Logging cancelled")

def async_function_from(output_function):
    async def async_output_function(deviceCount):
        measurement = aggregate_measurements(deviceCount)
        output_function(measurement)

    return async_output_function

def record_gpu_to(output_function, async_loop, deviceCount=1, polling_interval=1):
    asyncio.set_event_loop(async_loop)
    pynvml.nvmlInit()
    logger.info("Driver Version: {}".format(nativestr(pynvml.nvmlSystemGetDriverVersion())))
    deviceCount = pynvml.nvmlDeviceGetCount()
    async_loop.run_until_complete(output_function)
    print("Shutting down driver")
    pynvml.nvmlShutdown()

def start_record_gpu_to(output_function):
    new_loop = asyncio.new_event_loop()
    async_output_func = async_function_from(output_function)
    task_task = new_loop.create_task(record_measurements(async_output_func, 4, polling_interval=1))
    t = Thread(target=record_gpu_to, args=(task_task, new_loop))
    t.start()
    return t, new_loop, task_task

def main():

    try:
        t, loop, task_task = start_record_gpu_to(print)
        time.sleep(10)
    except KeyboardInterrupt:
        print("Cancelling")
        task_task.cancel()
    # loop.stop()
    # loop.close()
    # finally:
    #     print("Closing")
    #     loop.stop()
    #     loop.close()

if __name__=="__main__":
    main()<commit_msg>Testing logging<commit_after>import asyncio
import logging
import time
from concurrent.futures import CancelledError
from datetime import datetime
from threading import Thread

import pynvml
from toolz.functoolz import compose

logger = logging.getLogger(__name__)
logging.basicConfig(level=logging.DEBUG)
logger.setLevel(logging.DEBUG)

def nativestr(s):
    if isinstance(s, str):
        return s
    return s.decode('utf-8', 'replace')

def device_name_for(device_handle):
    """Get GPU device name"""
    try:
        return nativestr(pynvml.nvmlDeviceGetName(device_handle))
    except pynvml.NVMlError:
        return "NVIDIA"

def mem_for(device_handle):
    """Get GPU device memory consumption in percent"""
    try:
        memory_info = pynvml.nvmlDeviceGetMemoryInfo(device_handle)
        return memory_info.used * 100.0 / memory_info.total
    except pynvml.NVMLError:
        return None

def utilization_for(device_handle):
    """Get GPU device consumption in percent"""
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).gpu
    except pynvml.NVMLError:
        return None

def mem_utilization_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetUtilizationRates(device_handle).memory
    except pynvml.NVMLError:
        return None

def power_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetPowerUsage(device_handle)
    except pynvml.NVMLError:
        return None

def temperature_for(device_handle):
    try:
        return pynvml.nvmlDeviceGetTemperature(device_handle, 0)
    except pynvml.NVMLError:
        return None

_MEASUREMENTS_FUNCS ={
    "Name": device_name_for,
    "Memory": mem_for,
    "Utilization": utilization_for,
    "Memory Utilization": mem_utilization_for,
    "Power": power_for,
    "Temperature": temperature_for,
}

def measurements_for(gpu_handle):
    mes_dict = {k: func(gpu_handle) for k, func in _MEASUREMENTS_FUNCS.items()}
    mes_dict['timestamp'] = str(datetime.now())
    return mes_dict

async def aggregate_measurements(device_count):
    measures_for_device = compose(measurements_for,
                                  pynvml.nvmlDeviceGetHandleByIndex)
    return {i:measures_for_device(i) for i in range(device_count)}

async def record_measurements(async_reporting_func, polling_interval=1):
    try:
        deviceCount = pynvml.nvmlDeviceGetCount()
        while True:
            measurement = await aggregate_measurements(deviceCount)
            await async_reporting_func(measurement)
            await asyncio.sleep(polling_interval)
    except CancelledError:  # TODO: Better control for aync loop
        logger.info("Logging cancelled")

def async_function_from(output_function):
    async def async_output_function(measurement):
        output_function(measurement)
    return async_output_function

def record_gpu_to(output_function, async_loop, deviceCount=1, polling_interval=1):
    asyncio.set_event_loop(async_loop)
    pynvml.nvmlInit()
    logger.info("Driver Version: {}".format(nativestr(pynvml.nvmlSystemGetDriverVersion())))
    async_loop.run_until_complete(output_function)
    logger.info("Shutting down driver")
    pynvml.nvmlShutdown()

def start_record_gpu_to(output_function):
    new_loop = asyncio.new_event_loop()
    task = new_loop.create_task(record_measurements(async_function_from(output_function), polling_interval=1))
    t = Thread(target=record_gpu_to, args=(task, new_loop))
    t.start()
    return task

def main():

    try:
        t, loop, task_task = start_record_gpu_to(print)
        time.sleep(10)
    except KeyboardInterrupt:
        logger.info("Cancelling")
        task_task.cancel()
    # loop.stop()
    # loop.close()
    # finally:
    #     print("Closing")
    #     loop.stop()
    #     loop.close()

if __name__=="__main__":
    main()<|endoftext|>


<commit_before>from .variational_strategy import VariationalStrategy
from .mvn_variational_strategy import MVNVariationalStrategy

__all__ = [
    VariationalStrategy,
    MVNVariationalStrategy,
]
<commit_msg>Fix lint<commit_after>from .variational_strategy import VariationalStrategy
from .mvn_variational_strategy import MVNVariationalStrategy

__all__ = [
    VariationalStrategy,
    MVNVariationalStrategy,
]
<|endoftext|>