<commit_before>import logging
import multiprocessing
import re
import subprocess
import threading
import zmq
from zmq.utils.strtypes import b, u

LOG = logging.getLogger("ROB.lib.reactor")

class Matcher(object):
    def __init__(self, rule):
        self.name = rule["name"]
        self.field = rule["match"]["field"]
        self.pattern = re.compile(rule["match"]["pattern"])
        self.bin = rule["exec"]["path"]
        self.timeout = rule["exec"]["timeout"]
        self.args = rule["exec"]["args"]

    def match(self, topic, uuid, data):
        if self.field == "topic":
            return self.pattern.match(topic) is not None
        else:
            return self.pattern.match(uuid) is not None

    def run(self, topic, uuid, data):
        variables = {"topic": topic, "uuid": uuid}
        args = [self.bin]
        for arg in self.args:
            if arg[0] == '$':
                args.append(variables[arg[1:]])
            else:
                args.append(arg)

        LOG.debug("Running: %s", args)
        try:
            out = subprocess.check_output(args, stderr=subprocess.STDOUT,
                                          timeout=self.timeout)
        except OSError as err:
            LOG.error("Unable to run %s (%s)", args, err)
        except subprocess.TimeoutExpired:
            LOG.error("Timeout when running %s", args)
        else:
            LOG.debug("=> %s", out)

class Worker(threading.Thread):
    def __init__(self, matchers):
        super().__init__()
        self.matchers = matchers

    def run(self):
        self.context = zmq.Context.instance()
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.connect("inproc://workers")
        while True:
            msg = self.sock.recv_multipart()
            matcher_index = int(msg[0])
            (topic, uuid, data) = msg[1:]
            LOG.debug("Running matcher n°%d on %s", matcher_index, self.name)
            self.matchers[matcher_index].run(topic, uuid, data)

class Reactor(multiprocessing.Process):
    def __init__(self, options, outbound):
        super().__init__()
        self.options = options
        self.matchers = []
        self.outbound = outbound
        self.workers = []

    def setup(self):
        # Connect to the stream
        self.context = zmq.Context.instance()
        self.sub = self.context.socket(zmq.SUB)
        self.sub.setsockopt(zmq.SUBSCRIBE, b"")
        LOG.debug("Connecting to outbound (%s)", self.outbound)
        self.sub.connect(self.outbound)

        # Loop on all rules
        for rule in self.options["rules"]:
            m = Matcher(rule)
            self.matchers.append(m)

        # Create the workers
        self.ctrl = self.context.socket(zmq.DEALER)
        self.ctrl.bind("inproc://workers")

        LOG.debug("Starting %d workers", self.options["workers"])
        for _ in range(0, self.options["workers"]):
            w = Worker(self.matchers)
            w.start()
            self.workers.append(w)

    def run(self):
        self.setup()
        while True:
            msg = self.sub.recv_multipart()
            try:
                (topic, uuid, dt, data) = (u(m) for m in msg)
            except (IndexError, ValueError):
                LOG.error("Invalid message: %s", msg)
                continue

            for (i, m) in enumerate(self.matchers):
                if m.match(topic, uuid, data):
                    LOG.debug("%s matching %s", msg, m.name)
                    self.ctrl.send_multipart([b(str(i)),
                                              b(topic),
                                              b(uuid),
                                              b(dt),
                                              b(data)])
<commit_msg>Reactor: allow to match on each field<commit_after>import logging
import multiprocessing
import re
import subprocess
import threading
import zmq
from zmq.utils.strtypes import b, u

LOG = logging.getLogger("ROB.lib.reactor")

class Matcher(object):
    def __init__(self, rule):
        self.name = rule["name"]
        self.field = rule["match"]["field"]
        self.pattern = re.compile(rule["match"]["pattern"])
        self.bin = rule["exec"]["path"]
        self.timeout = rule["exec"]["timeout"]
        self.args = rule["exec"]["args"]

    def match(self, variables):
        return self.pattern.match(variables[self.field]) is not None

    def run(self, topic, uuid, datetime, username, data):
        variables = {"topic": topic,
                     "uuid": uuid,
                     "datetime": datetime,
                     "username": username}
        args = [self.bin]
        for arg in self.args:
            if arg[0] == '$':
                # TODO: check for errors
                args.append(variables[arg[1:]])
            else:
                args.append(arg)

        LOG.debug("Running: %s", args)
        try:
            out = subprocess.check_output(args, stderr=subprocess.STDOUT,
                                          timeout=self.timeout)
        except OSError as err:
            LOG.error("Unable to run %s (%s)", args, err)
        except subprocess.TimeoutExpired:
            LOG.error("Timeout when running %s", args)
        else:
            LOG.debug("=> %s", out)

class Worker(threading.Thread):
    def __init__(self, matchers):
        super().__init__()
        self.matchers = matchers

    def run(self):
        self.context = zmq.Context.instance()
        self.sock = self.context.socket(zmq.DEALER)
        self.sock.connect("inproc://workers")
        while True:
            msg = self.sock.recv_multipart()
            matcher_index = int(msg[0])
            (topic, uuid, dt, username, data) = msg[1:]
            LOG.debug("Running matcher n°%d on %s", matcher_index, self.name)
            self.matchers[matcher_index].run(topic, uuid, dt, username, data)

class Reactor(multiprocessing.Process):
    def __init__(self, options, outbound):
        super().__init__()
        self.options = options
        self.matchers = []
        self.outbound = outbound
        self.workers = []

    def setup(self):
        # Connect to the stream
        self.context = zmq.Context.instance()
        self.sub = self.context.socket(zmq.SUB)
        self.sub.setsockopt(zmq.SUBSCRIBE, b"")
        LOG.debug("Connecting to outbound (%s)", self.outbound)
        self.sub.connect(self.outbound)

        # Loop on all rules
        for rule in self.options["rules"]:
            m = Matcher(rule)
            self.matchers.append(m)

        # Create the workers
        self.ctrl = self.context.socket(zmq.DEALER)
        self.ctrl.bind("inproc://workers")

        LOG.debug("Starting %d workers", self.options["workers"])
        for _ in range(0, self.options["workers"]):
            w = Worker(self.matchers)
            w.start()
            self.workers.append(w)

    def run(self):
        self.setup()
        while True:
            msg = self.sub.recv_multipart()
            try:
                (topic, uuid, datetime, username, data) = (u(m) for m in msg)
            except (IndexError, ValueError):
                LOG.error("Invalid message: %s", msg)
                continue

            variables = {"topic": topic,
                         "uuid": uuid,
                         "datetime": datetime,
                         "username": username}
            for (i, m) in enumerate(self.matchers):
                if m.match(variables):
                    LOG.debug("%s matching %s", msg, m.name)
                    self.ctrl.send_multipart([b(str(i)),
                                              b(topic),
                                              b(uuid),
                                              b(datetime),
                                              b(username),
                                              b(data)])
<|endoftext|>


<commit_before>import unittest
from datetime import date

from mock import patch

from app.brain.exercises_management.rep_exercises_management import RepExercisesManagement
from app.models import RepExercisesTaxonomy, RepExercisesHistory

class RepExercisesManagementTests(unittest.TestCase):
    def setUp(self):
        self.exercises = [
            RepExercisesTaxonomy(
                name='c_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='e_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=True,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='b_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=True,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='a_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=True,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='d_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=True,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            )
        ]
        self.exercises[0].id = 1
        self.exercises[1].id = 2
        self.exercises[2].id = 3
        self.exercises[3].id = 4
        self.exercises[4].id = 5

    @patch(
        'app.brain.exercises_management.rep_exercises_management.RepExercisesTaxonomyService.get_list_of_all_exercises')
    def test_get_valid_id_exercise_pairs(self, taxonomy_service_mock):
        taxonomy_service_mock.return_value = self.exercises
        expected_results = [('4', 'a_exercise'), ('3', 'b_exercise'), ('1', 'c_exercise'), ('5', 'd_exercise'),
                            ('2', 'e_exercise')]
        actual_results = RepExercisesManagement.get_valid_id_exercise_pairs()
        self.assertListEqual(actual_results, expected_results)

    @patch('app.brain.exercises_management.rep_exercises_management.RepExercisesHistoryService.add_entry_to_db')
    def test_submit_history_entry(self, db_mock):
        expected_date = date(year=2000, month=12, day=23)
        expected_result = RepExercisesHistory(
            user_id=3,
            exercise_id=12,
            sets=3,
            reps=12,
            weight=12.5,
            date=expected_date
        )
        actual_result = RepExercisesManagement.submit_history_entry(user_id=3,
                                                                    exercise_id=12,
                                                                    sets=3,
                                                                    reps=12,
                                                                    weight=12.5,
                                                                    exercise_date=expected_date
                                                                    )
        self.assertEqual(actual_result.user_id, expected_result.user_id)
        self.assertEqual(actual_result.exercise_id, expected_result.exercise_id)
        self.assertEqual(actual_result.sets, expected_result.sets)
        self.assertEqual(actual_result.reps, expected_result.reps)
        self.assertEqual(actual_result.weight, expected_result.weight)
        self.assertEqual(actual_result.date, expected_result.date)
<commit_msg>Unit test written<commit_after>import unittest
from datetime import date

from mock import patch

from app.brain.exercises_management.rep_exercises_management import RepExercisesManagement
from app.models import RepExercisesTaxonomy, RepExercisesHistory

class RepExercisesManagementTests(unittest.TestCase):
    def setUp(self):
        self.exercises = [
            RepExercisesTaxonomy(
                name='c_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='e_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=True,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='b_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=True,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='a_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=True,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=False,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            ),
            RepExercisesTaxonomy(
                name='d_exercise',
                is_back=True,
                is_chest=True,
                is_shoulders=False,
                is_biceps=True,
                is_triceps=True,
                is_legs=False,
                is_core=True,
                is_balance=False,
                is_cardio=False,
                is_weight_per_hand=True
            )
        ]
        self.exercises[0].id = 1
        self.exercises[1].id = 2
        self.exercises[2].id = 3
        self.exercises[3].id = 4
        self.exercises[4].id = 5

    @patch(
        'app.brain.exercises_management.rep_exercises_management.RepExercisesTaxonomyService.get_list_of_all_exercises')
    def test_get_valid_id_exercise_pairs(self, taxonomy_service_mock):
        taxonomy_service_mock.return_value = self.exercises
        expected_results = [('4', 'a_exercise'), ('3', 'b_exercise'), ('1', 'c_exercise'), ('5', 'd_exercise'),
                            ('2', 'e_exercise')]
        actual_results = RepExercisesManagement.get_valid_id_exercise_pairs()
        self.assertListEqual(actual_results, expected_results)

    @patch('app.brain.exercises_management.rep_exercises_management.RepExercisesHistoryService.add_entry_to_db')
    def test_submit_history_entry(self, db_mock):
        expected_date = date(year=2000, month=12, day=23)
        expected_result = RepExercisesHistory(
            user_id=3,
            exercise_id=12,
            sets=3,
            reps=12,
            weight=12.5,
            date=expected_date
        )
        actual_result = RepExercisesManagement.submit_history_entry(user_id=3,
                                                                    exercise_id=12,
                                                                    sets=3,
                                                                    reps=12,
                                                                    weight=12.5,
                                                                    exercise_date=expected_date
                                                                    )
        self.assertEqual(actual_result.user_id, expected_result.user_id)
        self.assertEqual(actual_result.exercise_id, expected_result.exercise_id)
        self.assertEqual(actual_result.sets, expected_result.sets)
        self.assertEqual(actual_result.reps, expected_result.reps)
        self.assertEqual(actual_result.weight, expected_result.weight)
        self.assertEqual(actual_result.date, expected_result.date)

    @patch('app.brain.exercises_management.rep_exercises_management.RepExercisesTaxonomyService.add_entry_to_db')
    def test_submit_taxonomy_entry(self, db_mock):
        exercise_name = 'super_awesome_exercise'
        expected_entry = RepExercisesTaxonomy(
            name=exercise_name,
            is_back=True,
            is_chest=False,
            is_shoulders=False,
            is_biceps=True,
            is_triceps=True,
            is_legs=False,
            is_core=False,
            is_balance=True,
            is_cardio=True,
            is_weight_per_hand=True
        )

        actual_entry = RepExercisesManagement.submit_taxonomy_entry(
            name=exercise_name,
            is_back=True,
            is_chest=False,
            is_shoulders=False,
            is_biceps=True,
            is_triceps=True,
            is_legs=False,
            is_core=False,
            is_balance=True,
            is_cardio=True,
            is_weight_per_hand=True
        )

        self.assertEqual(actual_entry.name, expected_entry.name)
        self.assertEqual(actual_entry.is_back, expected_entry.is_back)
        self.assertEqual(actual_entry.is_chest, expected_entry.is_chest)
        self.assertEqual(actual_entry.is_shoulders, expected_entry.is_shoulders)
        self.assertEqual(actual_entry.is_biceps, expected_entry.is_biceps)
        self.assertEqual(actual_entry.is_triceps, expected_entry.is_triceps)
        self.assertEqual(actual_entry.is_legs, expected_entry.is_legs)
        self.assertEqual(actual_entry.is_core, expected_entry.is_core)
        self.assertEqual(actual_entry.is_balance, expected_entry.is_balance)
        self.assertEqual(actual_entry.is_cardio, expected_entry.is_cardio)
        self.assertEqual(actual_entry.is_weight_per_hand, expected_entry.is_weight_per_hand)
<|endoftext|>


<commit_before>import yaml
import os
import settings

def load_repository_info(image_name):
    with open(settings.IMAGES_DATA_PATH  , "r") as fh:
        repository_info = ([image_data for image_data in yaml.safe_load(fh) if image_data["name"] in args.image ] + [None])[0]
    if image:
        repository_info["builds"] = _load_builds(image_name)

def _load_builds(image_name):
    return [_load_build_info(image_name, build_id) for build_id in os.listdir(settings.BUILD_LOGS_PATH)]

def _load_build(image_name, build_id):
    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/build-data.yml", "r") as fh:
            build_info = yaml.safe_load(fh)
            build_info["id"] = build_id
    except:
        build_info = {
            "id": build_id,
            "success": False
        }

    return build_info

def load_build_logs(image_name, build_id):
    build_logs = {
        "out": None,
        "err": None
    }

    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/out.log", "r") as fh:
            build_logs["out"] = fh.read()
    except:
        pass

    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/err.log", "r") as fh:
            build_logs["err"] = fh.read()
    except:
        pass

    return build_logs
<commit_msg>fix image name condition<commit_after>import yaml
import os
import settings

def load_repository_info(image_name):
    with open(settings.IMAGES_DATA_PATH  , "r") as fh:
        repository_info = ([image_data for image_data in yaml.safe_load(fh) if image_data["name"] == image_name ] + [None])[0]
    if image:
        repository_info["builds"] = _load_builds(image_name)

def _load_builds(image_name):
    return [_load_build_info(image_name, build_id) for build_id in os.listdir(settings.BUILD_LOGS_PATH)]

def _load_build(image_name, build_id):
    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/build-data.yml", "r") as fh:
            build_info = yaml.safe_load(fh)
            build_info["id"] = build_id
    except:
        build_info = {
            "id": build_id,
            "success": False
        }

    return build_info

def load_build_logs(image_name, build_id):
    build_logs = {
        "out": None,
        "err": None
    }

    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/out.log", "r") as fh:
            build_logs["out"] = fh.read()
    except:
        pass

    try:
        with open(settings.BUILD_LOGS_PATH + "/" + image_name + "/" + build_id + "/err.log", "r") as fh:
            build_logs["err"] = fh.read()
    except:
        pass

    return build_logs
<|endoftext|>


<commit_before># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Train a Fast R-CNN network."""

import caffe
from fast_rcnn.config import cfg
import roi_data_layer.roidb as rdl_roidb
from utils.timer import Timer
import numpy as np
import os

from caffe.proto import caffe_pb2
import google.protobuf as pb2

class SolverWrapper(object):
    """A simple wrapper around Caffe's solver.
    This wrapper gives us control over he snapshotting process, which we
    use to unnormalize the learned bounding-box regression weights.
    """

    def __init__(self, solver_prototxt, roidb, output_dir,
                 pretrained_model=None):
        """Initialize the SolverWrapper."""
        self.output_dir = output_dir

        if (cfg.TRAIN.HAS_RPN and cfg.TRAIN.BBOX_REG and
            cfg.TRAIN.BBOX_NORMALIZE_TARGETS):
            # RPN can only use precomputed normalization because there are no
            # fixed statistics to compute a priori
            assert cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED

        if cfg.TRAIN.BBOX_REG:
            print 'Computing bounding-box regression targets...'
            self.bbox_means, self.bbox_stds = \
                    rdl_roidb.add_bbox_regression_targets(roidb)
            print 'done'

        self.solver = caffe.SGDSolver(solver_prototxt)
        if pretrained_model is not None:
            print ('Loading pretrained model '
                   'weights from {:s}').format(pretrained_model)
            self.solver.net.copy_from(pretrained_model)

        self.solver_param = caffe_pb2.SolverParameter()
        with open(solver_prototxt, 'rt') as f:
            pb2.text_format.Merge(f.read(), self.solver_param)

        self.solver.net.layers[0].set_roidb(roidb)

    def snapshot(self):
        """Take a snapshot of the network after unnormalizing the learned
        bounding-box regression weights. This enables easy use at test-time.
        """
        net = self.solver.net

        scale_bbox_params_faster_rcnn = (cfg.TRAIN.BBOX_REG and
                             cfg.TRAIN.BBOX_NORMALIZE_TARGETS and
                             net.params.has_key('bbox_pred'))

        scale_bbox_params_rfcn = (cfg.TRAIN.BBOX_REG and
                             cfg.TRAIN.BBOX_NORMALIZE_TARGETS and
                             net.params.has_key('rfcn_bbox'))

        scale_bbox_params_rpn = (cfg.TRAIN.RPN_NORMALIZE_TARGETS and
                                 net.params.has_key('rpn_bbox_pred'))

        if scale_bbox_params_faster_rcnn:
            # save original values
            orig_0 = net.params['bbox_pred'][0].data.copy()
            orig_1 = net.params['bbox_pred'][1].data.copy()

            # scale and shift with bbox reg unnormalization; then save snapshot
            net.params['bbox_pred'][0].data[...] = \
                    (net.params['bbox_pred'][0].data *
                     self.bbox_stds[:, np.newaxis])
            net.params['bbox_pred'][1].data[...] = \
                    (net.params['bbox_pred'][1].data *
                     self.bbox_stds + self.bbox_means)

        if scale_bbox_params_rpn:
            rpn_orig_0 = net.params['rpn_bbox_pred'][0].data.copy()
            rpn_orig_1 = net.params['rpn_bbox_pred'][1].data.copy()
            num_anchor = rpn_orig_0.shape[0] / 4
            # scale and shift with bbox reg unnormalization; then save snapshot
            self.rpn_means = np.tile(np.asarray(cfg.TRAIN.RPN_NORMALIZE_MEANS),
                                      num_anchor)
            self.rpn_stds = np.tile(np.asarray(cfg.TRAIN.RPN_NORMALIZE_STDS),
                                     num_anchor)
            net.params['rpn_bbox_pred'][0].data[...] = \
                (net.params['rpn_bbox_pred'][0].data *
                 self.rpn_stds[:, np.newaxis, np.newaxis, np.newaxis])
            net.params['rpn_bbox_pred'][1].data[...] = \
                (net.params['rpn_bbox_pred'][1].data *
                 self.rpn_stds + self.rpn_means)

        if scale_bbox_params_rfcn:
            # save original values
            orig_0 = net.params['rfcn_bbox'][0].data.copy()
            orig_1 = net.params['rfcn_bbox'][1].data.copy()
            repeat = orig_1.shape[0] / self.bbox_means.shape[0]

            # scale and shift with bbox reg unnormalization; then save snapshot
            net.params['rfcn_bbox'][0].data[...] = \
                    (net.params['rfcn_bbox'][0].data *
                     np.repeat(self.bbox_stds, repeat).reshape((orig_1.shape[0], 1, 1, 1)))
            net.params['rfcn_bbox'][1].data[...] = \
                    (net.params['rfcn_bbox'][1].data *
                     np.repeat(self.bbox_stds, repeat) + np.repeat(self.bbox_means, repeat))

        infix = ('_' + cfg.TRAIN.SNAPSHOT_INFIX
                 if cfg.TRAIN.SNAPSHOT_INFIX != '' else '')
        filename = (self.solver_param.snapshot_prefix + infix +
                    '_iter_{:d}'.format(self.solver.iter) + '.caffemodel')
        filename = os.path.join(self.output_dir, filename)
        net.save(str(filename))
        print 'Wrote snapshot to: {:s}'.format(filename)

        if scale_bbox_params_faster_rcnn:
            # restore net to original state
            net.params['bbox_pred'][0].data[...] = orig_0
            net.params['bbox_pred'][1].data[...] = orig_1
        if scale_bbox_params_rfcn:
            # restore net to original state
            net.params['rfcn_bbox'][0].data[...] = orig_0
            net.params['rfcn_bbox'][1].data[...] = orig_1
        if scale_bbox_params_rpn:
            # restore net to original state
            net.params['rpn_bbox_pred'][0].data[...] = rpn_orig_0
            net.params['rpn_bbox_pred'][1].data[...] = rpn_orig_1

        return filename

    def train_model(self, max_iters):
        """Network training loop."""
        last_snapshot_iter = -1
        timer = Timer()
        model_paths = []
        while self.solver.iter < max_iters:
            # Make one SGD update
            timer.tic()
            self.solver.step(1)
            timer.toc()
            if self.solver.iter % (10 * self.solver_param.display) == 0:
                print 'speed: {:.3f}s / iter'.format(timer.average_time)
                print 'fps: {:.3f}'.format(1.0 / timer.average_time)

            if self.solver.iter % cfg.TRAIN.SNAPSHOT_ITERS == 0:
                last_snapshot_iter = self.solver.iter
                model_paths.append(self.snapshot())

        if last_snapshot_iter != self.solver.iter:
            model_paths.append(self.snapshot())
        return model_paths

def get_training_roidb(imdb):
    """Returns a roidb (Region of Interest database) for use in training."""
    if cfg.TRAIN.USE_FLIPPED:
        print 'Appending horizontally-flipped training examples...'
        imdb.append_flipped_images()
        print 'done'

    print 'Preparing training data...'
    rdl_roidb.prepare_roidb(imdb)
    print 'done'

    return imdb.roidb

def filter_roidb(roidb):
    """Remove roidb entries that have no usable RoIs."""

    def is_valid(entry):
        # Valid images have:
        #   (1) At least one foreground RoI OR
        #   (2) At least one background RoI
        overlaps = entry['max_overlaps']
        # find boxes with sufficient overlap
        fg_inds = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0]
        # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
        bg_inds = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) &
                           (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
        # image is only valid if such boxes exist
        valid = len(fg_inds) > 0 or len(bg_inds) > 0
        return valid

    num = len(roidb)
    filtered_roidb = [entry for entry in roidb if is_valid(entry)]
    num_after = len(filtered_roidb)
    print 'Filtered {} roidb entries: {} -> {}'.format(num - num_after,
                                                       num, num_after)
    return filtered_roidb

def train_net(solver_prototxt, roidb, output_dir,
              pretrained_model=None, max_iters=40000):
    """Train a Fast R-CNN network."""

    roidb = filter_roidb(roidb)
    sw = SolverWrapper(solver_prototxt, roidb, output_dir,
                       pretrained_model=pretrained_model)

    print 'Solving...'
    model_paths = sw.train_model(max_iters)
    print 'done solving'
    return model_paths
<commit_msg>add protobuf text_format in train.py to avoid no module issue in some env<commit_after># --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------

"""Train a Fast R-CNN network."""

import caffe
from fast_rcnn.config import cfg
import roi_data_layer.roidb as rdl_roidb
from utils.timer import Timer
import numpy as np
import os

from caffe.proto import caffe_pb2
import google.protobuf as pb2

import google.protobuf.text_format

class SolverWrapper(object):
    """A simple wrapper around Caffe's solver.
    This wrapper gives us control over he snapshotting process, which we
    use to unnormalize the learned bounding-box regression weights.
    """

    def __init__(self, solver_prototxt, roidb, output_dir,
                 pretrained_model=None):
        """Initialize the SolverWrapper."""
        self.output_dir = output_dir

        if (cfg.TRAIN.HAS_RPN and cfg.TRAIN.BBOX_REG and
            cfg.TRAIN.BBOX_NORMALIZE_TARGETS):
            # RPN can only use precomputed normalization because there are no
            # fixed statistics to compute a priori
            assert cfg.TRAIN.BBOX_NORMALIZE_TARGETS_PRECOMPUTED

        if cfg.TRAIN.BBOX_REG:
            print 'Computing bounding-box regression targets...'
            self.bbox_means, self.bbox_stds = \
                    rdl_roidb.add_bbox_regression_targets(roidb)
            print 'done'

        self.solver = caffe.SGDSolver(solver_prototxt)
        if pretrained_model is not None:
            print ('Loading pretrained model '
                   'weights from {:s}').format(pretrained_model)
            self.solver.net.copy_from(pretrained_model)

        self.solver_param = caffe_pb2.SolverParameter()
        with open(solver_prototxt, 'rt') as f:
            pb2.text_format.Merge(f.read(), self.solver_param)

        self.solver.net.layers[0].set_roidb(roidb)

    def snapshot(self):
        """Take a snapshot of the network after unnormalizing the learned
        bounding-box regression weights. This enables easy use at test-time.
        """
        net = self.solver.net

        scale_bbox_params_faster_rcnn = (cfg.TRAIN.BBOX_REG and
                             cfg.TRAIN.BBOX_NORMALIZE_TARGETS and
                             net.params.has_key('bbox_pred'))

        scale_bbox_params_rfcn = (cfg.TRAIN.BBOX_REG and
                             cfg.TRAIN.BBOX_NORMALIZE_TARGETS and
                             net.params.has_key('rfcn_bbox'))

        scale_bbox_params_rpn = (cfg.TRAIN.RPN_NORMALIZE_TARGETS and
                                 net.params.has_key('rpn_bbox_pred'))

        if scale_bbox_params_faster_rcnn:
            # save original values
            orig_0 = net.params['bbox_pred'][0].data.copy()
            orig_1 = net.params['bbox_pred'][1].data.copy()

            # scale and shift with bbox reg unnormalization; then save snapshot
            net.params['bbox_pred'][0].data[...] = \
                    (net.params['bbox_pred'][0].data *
                     self.bbox_stds[:, np.newaxis])
            net.params['bbox_pred'][1].data[...] = \
                    (net.params['bbox_pred'][1].data *
                     self.bbox_stds + self.bbox_means)

        if scale_bbox_params_rpn:
            rpn_orig_0 = net.params['rpn_bbox_pred'][0].data.copy()
            rpn_orig_1 = net.params['rpn_bbox_pred'][1].data.copy()
            num_anchor = rpn_orig_0.shape[0] / 4
            # scale and shift with bbox reg unnormalization; then save snapshot
            self.rpn_means = np.tile(np.asarray(cfg.TRAIN.RPN_NORMALIZE_MEANS),
                                      num_anchor)
            self.rpn_stds = np.tile(np.asarray(cfg.TRAIN.RPN_NORMALIZE_STDS),
                                     num_anchor)
            net.params['rpn_bbox_pred'][0].data[...] = \
                (net.params['rpn_bbox_pred'][0].data *
                 self.rpn_stds[:, np.newaxis, np.newaxis, np.newaxis])
            net.params['rpn_bbox_pred'][1].data[...] = \
                (net.params['rpn_bbox_pred'][1].data *
                 self.rpn_stds + self.rpn_means)

        if scale_bbox_params_rfcn:
            # save original values
            orig_0 = net.params['rfcn_bbox'][0].data.copy()
            orig_1 = net.params['rfcn_bbox'][1].data.copy()
            repeat = orig_1.shape[0] / self.bbox_means.shape[0]

            # scale and shift with bbox reg unnormalization; then save snapshot
            net.params['rfcn_bbox'][0].data[...] = \
                    (net.params['rfcn_bbox'][0].data *
                     np.repeat(self.bbox_stds, repeat).reshape((orig_1.shape[0], 1, 1, 1)))
            net.params['rfcn_bbox'][1].data[...] = \
                    (net.params['rfcn_bbox'][1].data *
                     np.repeat(self.bbox_stds, repeat) + np.repeat(self.bbox_means, repeat))

        infix = ('_' + cfg.TRAIN.SNAPSHOT_INFIX
                 if cfg.TRAIN.SNAPSHOT_INFIX != '' else '')
        filename = (self.solver_param.snapshot_prefix + infix +
                    '_iter_{:d}'.format(self.solver.iter) + '.caffemodel')
        filename = os.path.join(self.output_dir, filename)
        net.save(str(filename))
        print 'Wrote snapshot to: {:s}'.format(filename)

        if scale_bbox_params_faster_rcnn:
            # restore net to original state
            net.params['bbox_pred'][0].data[...] = orig_0
            net.params['bbox_pred'][1].data[...] = orig_1
        if scale_bbox_params_rfcn:
            # restore net to original state
            net.params['rfcn_bbox'][0].data[...] = orig_0
            net.params['rfcn_bbox'][1].data[...] = orig_1
        if scale_bbox_params_rpn:
            # restore net to original state
            net.params['rpn_bbox_pred'][0].data[...] = rpn_orig_0
            net.params['rpn_bbox_pred'][1].data[...] = rpn_orig_1

        return filename

    def train_model(self, max_iters):
        """Network training loop."""
        last_snapshot_iter = -1
        timer = Timer()
        model_paths = []
        while self.solver.iter < max_iters:
            # Make one SGD update
            timer.tic()
            self.solver.step(1)
            timer.toc()
            if self.solver.iter % (10 * self.solver_param.display) == 0:
                print 'speed: {:.3f}s / iter'.format(timer.average_time)
                print 'fps: {:.3f}'.format(1.0 / timer.average_time)

            if self.solver.iter % cfg.TRAIN.SNAPSHOT_ITERS == 0:
                last_snapshot_iter = self.solver.iter
                model_paths.append(self.snapshot())

        if last_snapshot_iter != self.solver.iter:
            model_paths.append(self.snapshot())
        return model_paths

def get_training_roidb(imdb):
    """Returns a roidb (Region of Interest database) for use in training."""
    if cfg.TRAIN.USE_FLIPPED:
        print 'Appending horizontally-flipped training examples...'
        imdb.append_flipped_images()
        print 'done'

    print 'Preparing training data...'
    rdl_roidb.prepare_roidb(imdb)
    print 'done'

    return imdb.roidb

def filter_roidb(roidb):
    """Remove roidb entries that have no usable RoIs."""

    def is_valid(entry):
        # Valid images have:
        #   (1) At least one foreground RoI OR
        #   (2) At least one background RoI
        overlaps = entry['max_overlaps']
        # find boxes with sufficient overlap
        fg_inds = np.where(overlaps >= cfg.TRAIN.FG_THRESH)[0]
        # Select background RoIs as those within [BG_THRESH_LO, BG_THRESH_HI)
        bg_inds = np.where((overlaps < cfg.TRAIN.BG_THRESH_HI) &
                           (overlaps >= cfg.TRAIN.BG_THRESH_LO))[0]
        # image is only valid if such boxes exist
        valid = len(fg_inds) > 0 or len(bg_inds) > 0
        return valid

    num = len(roidb)
    filtered_roidb = [entry for entry in roidb if is_valid(entry)]
    num_after = len(filtered_roidb)
    print 'Filtered {} roidb entries: {} -> {}'.format(num - num_after,
                                                       num, num_after)
    return filtered_roidb

def train_net(solver_prototxt, roidb, output_dir,
              pretrained_model=None, max_iters=40000):
    """Train a Fast R-CNN network."""

    roidb = filter_roidb(roidb)
    sw = SolverWrapper(solver_prototxt, roidb, output_dir,
                       pretrained_model=pretrained_model)

    print 'Solving...'
    model_paths = sw.train_model(max_iters)
    print 'done solving'
    return model_paths
<|endoftext|>


<commit_before>#! /usr/bin/env python3
#
# Extract gene mention candidates, add features, and
# perform distant supervision
#

import fileinput
import json
import random

from dstruct.Mention import Mention
from dstruct.Sentence import Sentence
from helper.dictionaries import load_dict
from helper.easierlife import get_all_phrases_in_sentence, get_dict_from_TSVline, TSVstring2list, no_op

RANDOM_EXAMPLES_PROB = 0.01
RANDOM_EXAMPLES_QUOTA = 1000
ACRONYMS_QUOTA = 1000
ACRONYMS_PROB = 0.01
false_acronyms = 0
random_examples = 0

DOC_ELEMENTS_OR_INDIVIDUALS =["figure", "table", "figures", "tables", "fig",
"fig.", "figs", "figs.", "individual", "individuals"]

## Perform the supervision
# We don't supervise anything as positive, except things that are in our
# collection of positive labelled examples because we have the geneRifs that
# will help us a lot since they are all positively labelled
def supervise(mention, sentence):
    # XXX (Matteo) The following commented code is from when we didn't use
    # geneRifs yet. It is inspired by similar code in the pharm repository.
    #
    # If the candidate mention a gene symbol in the supervision dictionary, and
    # not an English word, and not a medical acronym, and not a NIH or NSF
    # grant code, and not a Roman numeral then label it as correct, provided
    # we get head when we flip the coin
    #if random.random() < SUPERVISION_PROB and \
    #    mention_word.lower() not in english_dict and \
    #    mention_word not in med_acrons_dict and \
    #    mention_word not in nih_grants_dict and \
    #    mention_word not in nsf_grants_dict and \
    #    not re.match("^(IV|VI{,3}|I{1,4})$", mention_word):
    #        mention.is_correct = True
    #
    # Correct if it is in our collection of positive examples
    if frozenset([sentence.doc_id, str(sentence.sent_id), mention.entity]) in pos_mentions_dict or \
            frozenset([sentence.doc_id, str(sentence.sent_id), mention.words[0].word]) in pos_mentions_dict:
        mention.is_correct = True
    # Not correct if the previous word is one of the following keywords
    # denoting a figure, a table, or an individual
    if sentence.get_prev_wordobject(mention) != None and \
            sentence.get_prev_wordobject(mention).word.casefold() in DOC_ELEMENTS_OR_INDIVIDUALS:
        mention.is_correct = False
    # Not correct if it is in our collection of negative examples
    if frozenset([sentence.doc_id, str(sentence.sent_id), mention.entity]) in neg_mentions_dict or \
            frozenset([sentence.doc_id, str(sentence.sent_id), mention.words[0].word]) in neg_mentions_dict:
        mention.is_correct = False
    # If the sentence is less than 4 words, it probably doesn't contain
    # enough information to convey anything useful for the supervision, so let
    # the system learn what it is
    if len(sentence.words) < 4:
        mention.is_correct = None

## Add features to a gene mention
def add_features(mention, sentence):
    # The mention is a 'main' symbol:
    if len(mention.words) == 1 and mention.entity == mention.words[0].word:
        mention.add_feature('IS_MAIN_SYMBOL')
    # The mention is a synonym symbol
    # XXX (Matteo) this is not entirely foolproof
    elif len(mention.words) == 1 and not (mention.word[0].word.isalpha() and \
            mention.words[0].word.casefold() == mention.words[0].word):
        mention.add_feature('IS_SYNONYM_SYMBOL')
    # The mention is a long name
    else:
        mention.add_feature('IS_LONG_NAME')
    # The mention is a single word that is in the English dictionary
    if len(mention.words) == 1 and mention.words[0].word.casefold() in english_dict or \
            mention.words[0].lemma in english_dict:
        mention.add_feature('IS_ENGLISH_WORD')
    # The NER is an organization, or a location, or a person
    if len(mention.words) == 1 and mention.words[0].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("IS_" + mention.words[0].ner)
    # The word comes after an organization, or a location, or a person. We skip
    # commas as they may trick us
    comes_after = None
    idx = mention.wordidxs[0] - 1
    while idx >= 0 and sentence.words[idx].lemma == ",":
        idx -= 1
    if idx >= 0 and sentence.words[idx].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("COMES_AFTER_" + sentence.words[idx].ner)
        comes_after = sentence.words[idx].ner
    # The word comes before an organization, or a location, or a person. We
    # skip commas, as they may trick us.
    comes_before = None
    idx = mention.wordidxs[-1] + 1
    while idx < len(sentence.words) and sentence.words[idx].lemma == ",":
        idx += 1
    if idx < len(sentence.words) and sentence.words[idx].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("COMES_BEFORE_" + sentence.words[idx].ner)
        comes_before = sentence.words[idx].ner
    # The word is between two words that are an organization, or a location or a person
    if comes_before and comes_after:
        mention.add_feature("IS_BETWEEN_" + comes_after + "_" + comes_before)
    # The word comes after a "document element" (e.g., table, or figure)
    prev_word = sentence.get_prev_wordobject(mention)
    if prev_word != None and prev_word.word.casefold() in DOC_ELEMENTS_OR_INDIVIDUALS:
        mention.add_feature("IS_AFTER_DOC_ELEMENT")
    # The labels and the NERs on the shortest dependency path
    # between a verb and the mention word.
    minl = 100
    minp = None
    minw = None
    for word2 in sentence.words:
        if word2.pos.startswith('VB') and word2.lemma != 'be':
            p = sentence.get_word_dep_path(mention.wordidxs[0], word2.in_sent_idx)
            if len(p) < minl:
                minl = len(p)
                minp = p
                minw = word2.lemma
    if minw != None:
        mention.add_feature('EXT_VERB_PATH_[' + minw + ']' + minp)
        mention.add_feature('VERB_PATH_[' + minw + ']')
    # The keywords that appear in the sentence with the mention
    minl = 100
    minp = None
    minw = None
    for word2 in sentence.words:
        if word2.lemma in ["gene", "genes", "protein", "proteins", "DNA",
                "rRNA", "cell", "cells", "tumor", "tumors", "domain",
                "sequence", "sequences", "alignment", "expression", "mRNA",
                "knockout", "recruitment", "hybridization", "isoform",
                "chromosome"]:
            p = sentence.get_word_dep_path(mention.wordidxs[0], word2.in_sent_idx)
            mention.add_feature("KEYWORD_[" + word2.lemma + "]")
            if len(p) < minl:
                minl = len(p)
                minp = p
                minw = word2.lemma
    # Special feature for the keyword on the shortest dependency path
    if minw != None:
        mention.add_feature('EXT_KEYWORD_SHORTEST_PATH_[' + minw + ']' + minp)
        mention.add_feature('KEYWORD_SHORTEST_PATH_[' + minw + ']')
    # The lemma on the left of the mention, if present, provided it's not a
    # ",", in which case we get the previous word
    idx = mention.wordidxs[0] - 1
    while idx >= 0 and sentence.words[idx].lemma == ",":
        idx -= 1
    if idx >= 0:
        mention.add_feature("WINDOW_LEFT_1_[{}]".format(
            sentence.words[idx].lemma))
    # The word on the right of the mention, if present, provided it's not a
    # ",", in which case we get the next word.
    idx = mention.wordidxs[-1] + 1
    while idx < len(sentence.words) and sentence.words[idx].lemma == ",":
        idx += 1
    if idx < len(sentence.words):
        mention.add_feature("WINDOW_RIGHT_1_[{}]".format(
            sentence.words[idx].lemma))
    # The word appears many times (more than 4) in the sentence
    if len(mentions.words) == 1 and [w.word for w in sentence.words].count(mention.words[0].word) > 4:
        mention.add_feature("APPEARS_MANY_TIMES_IN_SENTENCE")
    # There are many PERSONs/ORGANIZATIONs/LOCATIONs in the sentence
    for ner in ["PERSON", "ORGANIZAZTION", "LOCATION"]:
        if [x.lemma for x in sentence.words].count(ner) > 4:
            mention.add_feature("MANY_{}_IN_SENTENCE".format(ner))

# Add features that are related to the entire set of mentions candidates
def add_features_to_all(mentions, sentence):
    # Number of other mentions in the sentence
    for i in range(len(mentions)):
        for mention in mentions:
            mention.add_feature("{}_OTHER_MENTIONS_IN_SENTENCE".format(i - 1))

## Return a list of mention candidates extracted from the sentence
def extract(sentence):
    mentions = []
    global random_examples
    # The following set keeps a list of indexes we already looked at and which
    # contained a mention
    history = set()
    words = sentence.words
    # Scan all subsequences of the sentence
    for start, end in get_all_phrases_in_sentence(sentence, max_mention_length):
        if start in history or end in history:
                continue
        phrase = " ".join([word.word for word in words[start:end]])
        phrase_caseless = phrase.casefold()
        mention = None
        # If the phrase is in the dictionary, then is a mention candidate
        if phrase_caseless in merged_genes_dict:
            mention = Mention("GENE",
                    "|".join(merged_genes_dict[phrase_caseless]), words[start:end])
            add_features(mention, sentence)
            mentions.append(mention)
        else: # Potentially generate a random mention
            # Check whether it's a number, we do not want to generate a mention
            # with it.
            try:
                float(word.word)
            except:
                is_number = False
            else:
                is_number = True
            if word.word.isalnum() and not is_number and word.word not in stopwords_dict and \
                not word.pos.startswith("VB") and word.word not in merged_genes_dict and \
                len(sentence.words) > 5 and random.random() < RANDOM_EXAMPLES_PROB and \
                random_examples < RANDOM_EXAMPLES_QUOTA:
                # Generate a mention that somewhat resembles what a gene may look like,
                # or at least its role in the sentence.
                # This mention is supervised (as false) in the code calling this function
                mention = Mention("RANDOM", word.word, [word,])
                add_features(mention, sentence)
                random_examples += 1
                mentions.append(mention)

# Load the dictionaries that we need
merged_genes_dict = load_dict("merged_genes")
english_dict = load_dict("english")
stopwords_dict = load_dict("stopwords")
pos_mentions_dict = load_dict("pos_gene_mentions")
neg_mentions_dict = load_dict("neg_gene_mentions")
# XXX (Matteo) This dictionaries were used when we didn't have geneRifs to
# label mention candidates as positive. Now they're no longer needed. See also
# comment in supervise().
#nih_grants_dict = load_dict("nih_grants")
#nsf_grants_dict = load_dict("nsf_grants")
#med_acrons_dict = load_dict("med_acrons")
max_mention_length = 0
for key in merged_genes_dict:
    length = len(key.split())
    if length > max_mention_length:
        max_mention_length = length
max_mention_length *= 2 #doubling to take into account commas and who knows what

if __name__ == "__main__":
    # Process the input
    with fileinput.input() as input_files:
        for line in input_files:
            # This is for json
            #line_dict = json.loads(line)
            # This is for tsv
            line_dict = get_dict_from_TSVline(line, ["doc_id", "sent_id", "wordidxs",
            "words", "poses", "ners", "lemmas", "dep_paths", "dep_parents",
            "bounding_boxes", "acronym"], [no_op, int, lambda x :
                TSVstring2list(x, int), TSVstring2list, TSVstring2list,
                TSVstring2list, TSVstring2list, TSVstring2list, lambda x :
                TSVstring2list(x, int), TSVstring2list, no_op])
            sentence = Sentence(line_dict["doc_id"], line_dict["sent_id"],
                    line_dict["wordidxs"], line_dict["words"],
                    line_dict["poses"], line_dict["ners"], line_dict["lemmas"],
                    line_dict["dep_paths"], line_dict["dep_parents"],
                    line_dict["bounding_boxes"])
            # Get list of mentions candidates in this sentence
            mentions = extract(sentence)
            # Add features that use information about other mentions
            if len(mentions) > 1:
                add_features_to_all(mentions, sentence)
            # Supervise according to the mention type
            for mention in mentions:
                if "acronym" in line_dict:
                    if mention.words[0].word == line_dict["acronym"]:
                        mention.type = "ACRONYM"
                        if false_acronyms < ACRONYMS_QUOTA and \
                        random.random() < ACRONYMS_PROB:
                            mention.is_correct = False
                            false_acronyms += 1
                elif mention.type == "RANDOM":
                    # this is a randomly generated example that we assume
                    # to be false
                    mention.is_correct = False
                else:
                    supervise(mention, sentence)
                # Print!
                # This is for json
                #print(mention.json_dump())
                # This is for TSV
                print(mention.tsv_dump())

<commit_msg>Bring fully to new world where a gene mention can have multiple words<commit_after>#! /usr/bin/env python3
#
# Extract gene mention candidates, add features, and
# perform distant supervision
#

import fileinput
import json
import random

from dstruct.Mention import Mention
from dstruct.Sentence import Sentence
from helper.dictionaries import load_dict
from helper.easierlife import get_all_phrases_in_sentence, get_dict_from_TSVline, TSVstring2list, no_op

RANDOM_EXAMPLES_PROB = 0.01
RANDOM_EXAMPLES_QUOTA = 1000
ACRONYMS_QUOTA = 1000
ACRONYMS_PROB = 0.01
false_acronyms = 0
random_examples = 0

DOC_ELEMENTS_OR_INDIVIDUALS =["figure", "table", "figures", "tables", "fig",
"fig.", "figs", "figs.", "individual", "individuals"]

## Perform the supervision
# We don't supervise anything as positive, except things that are in our
# collection of positive labelled examples because we have the geneRifs that
# will help us a lot since they are all positively labelled
def supervise(mention, sentence):
    # XXX (Matteo) The following commented code is from when we didn't use
    # geneRifs yet. It is inspired by similar code in the pharm repository.
    #
    # If the candidate mention a gene symbol in the supervision dictionary, and
    # not an English word, and not a medical acronym, and not a NIH or NSF
    # grant code, and not a Roman numeral then label it as correct, provided
    # we get head when we flip the coin
    #if random.random() < SUPERVISION_PROB and \
    #    mention_word.lower() not in english_dict and \
    #    mention_word not in med_acrons_dict and \
    #    mention_word not in nih_grants_dict and \
    #    mention_word not in nsf_grants_dict and \
    #    not re.match("^(IV|VI{,3}|I{1,4})$", mention_word):
    #        mention.is_correct = True
    #
    # Correct if it is in our collection of positive examples
    if frozenset([sentence.doc_id, str(sentence.sent_id), mention.entity]) in pos_mentions_dict or \
            frozenset([sentence.doc_id, str(sentence.sent_id), mention.words[0].word]) in pos_mentions_dict:
        mention.is_correct = True
    # Not correct if the previous word is one of the following keywords
    # denoting a figure, a table, or an individual
    if sentence.get_prev_wordobject(mention) != None and \
            sentence.get_prev_wordobject(mention).word.casefold() in DOC_ELEMENTS_OR_INDIVIDUALS:
        mention.is_correct = False
    # Not correct if it is in our collection of negative examples
    if frozenset([sentence.doc_id, str(sentence.sent_id), mention.entity]) in neg_mentions_dict or \
            frozenset([sentence.doc_id, str(sentence.sent_id), mention.words[0].word]) in neg_mentions_dict:
        mention.is_correct = False
    # If the sentence is less than 4 words, it probably doesn't contain
    # enough information to convey anything useful for the supervision, so let
    # the system learn what it is
    if len(sentence.words) < 4:
        mention.is_correct = None

## Add features to a gene mention
def add_features(mention, sentence):
    # The mention is a 'main' symbol:
    if len(mention.words) == 1 and mention.entity == mention.words[0].word:
        mention.add_feature('IS_MAIN_SYMBOL')
    # The mention is a synonym symbol
    # XXX (Matteo) this is not entirely foolproof
    elif len(mention.words) == 1 and not (mention.word[0].word.isalpha() and \
            mention.words[0].word.casefold() == mention.words[0].word):
        mention.add_feature('IS_SYNONYM_SYMBOL')
    # The mention is a long name
    else:
        mention.add_feature('IS_LONG_NAME')
    # The mention is a single word that is in the English dictionary
    if len(mention.words) == 1 and mention.words[0].word.casefold() in english_dict or \
            mention.words[0].lemma in english_dict:
        mention.add_feature('IS_ENGLISH_WORD')
    # The NER is an organization, or a location, or a person
    if len(mention.words) == 1 and mention.words[0].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("IS_" + mention.words[0].ner)
    # The word comes after an organization, or a location, or a person. We skip
    # commas as they may trick us
    comes_after = None
    idx = mention.wordidxs[0] - 1
    while idx >= 0 and sentence.words[idx].lemma == ",":
        idx -= 1
    if idx >= 0 and sentence.words[idx].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("COMES_AFTER_" + sentence.words[idx].ner)
        comes_after = sentence.words[idx].ner
    # The word comes before an organization, or a location, or a person. We
    # skip commas, as they may trick us.
    comes_before = None
    idx = mention.wordidxs[-1] + 1
    while idx < len(sentence.words) and sentence.words[idx].lemma == ",":
        idx += 1
    if idx < len(sentence.words) and sentence.words[idx].ner in [ "ORGANIZATION", "LOCATION", "PERSON"]:
        mention.add_feature("COMES_BEFORE_" + sentence.words[idx].ner)
        comes_before = sentence.words[idx].ner
    # The word is between two words that are an organization, or a location or a person
    if comes_before and comes_after:
        mention.add_feature("IS_BETWEEN_" + comes_after + "_" + comes_before)
    # The word comes after a "document element" (e.g., table, or figure)
    prev_word = sentence.get_prev_wordobject(mention)
    if prev_word != None and prev_word.word.casefold() in DOC_ELEMENTS_OR_INDIVIDUALS:
        mention.add_feature("IS_AFTER_DOC_ELEMENT")
    # The labels and the NERs on the shortest dependency path
    # between a verb and the mention word.
    minl = 100
    minp = None
    minw = None
    for word2 in sentence.words:
        if word2.pos.startswith('VB') and word2.lemma != 'be':
            p = sentence.get_word_dep_path(mention.wordidxs[0], word2.in_sent_idx)
            if len(p) < minl:
                minl = len(p)
                minp = p
                minw = word2.lemma
    if minw != None:
        mention.add_feature('EXT_VERB_PATH_[' + minw + ']' + minp)
        mention.add_feature('VERB_PATH_[' + minw + ']')
    # The keywords that appear in the sentence with the mention
    minl = 100
    minp = None
    minw = None
    for word2 in sentence.words:
        if word2.lemma in ["gene", "genes", "protein", "proteins", "DNA",
                "rRNA", "cell", "cells", "tumor", "tumors", "domain",
                "sequence", "sequences", "alignment", "expression", "mRNA",
                "knockout", "recruitment", "hybridization", "isoform",
                "chromosome"]:
            p = sentence.get_word_dep_path(mention.wordidxs[0], word2.in_sent_idx)
            mention.add_feature("KEYWORD_[" + word2.lemma + "]")
            if len(p) < minl:
                minl = len(p)
                minp = p
                minw = word2.lemma
    # Special feature for the keyword on the shortest dependency path
    if minw != None:
        mention.add_feature('EXT_KEYWORD_SHORTEST_PATH_[' + minw + ']' + minp)
        mention.add_feature('KEYWORD_SHORTEST_PATH_[' + minw + ']')
    # The lemma on the left of the mention, if present, provided it's not a
    # ",", in which case we get the previous word
    idx = mention.wordidxs[0] - 1
    while idx >= 0 and sentence.words[idx].lemma == ",":
        idx -= 1
    if idx >= 0:
        mention.add_feature("WINDOW_LEFT_1_[{}]".format(
            sentence.words[idx].lemma))
    # The word on the right of the mention, if present, provided it's not a
    # ",", in which case we get the next word.
    idx = mention.wordidxs[-1] + 1
    while idx < len(sentence.words) and sentence.words[idx].lemma == ",":
        idx += 1
    if idx < len(sentence.words):
        mention.add_feature("WINDOW_RIGHT_1_[{}]".format(
            sentence.words[idx].lemma))
    # The word appears many times (more than 4) in the sentence
    if len(mentions.words) == 1 and [w.word for w in sentence.words].count(mention.words[0].word) > 4:
        mention.add_feature("APPEARS_MANY_TIMES_IN_SENTENCE")
    # There are many PERSONs/ORGANIZATIONs/LOCATIONs in the sentence
    for ner in ["PERSON", "ORGANIZAZTION", "LOCATION"]:
        if [x.lemma for x in sentence.words].count(ner) > 4:
            mention.add_feature("MANY_{}_IN_SENTENCE".format(ner))

# Add features that are related to the entire set of mentions candidates
def add_features_to_all(mentions, sentence):
    # Number of other mentions in the sentence
    for i in range(len(mentions)):
        for mention in mentions:
            mention.add_feature("{}_OTHER_MENTIONS_IN_SENTENCE".format(i - 1))

## Return a list of mention candidates extracted from the sentence
def extract(sentence):
    mentions = []
    global random_examples
    # The following set keeps a list of indexes we already looked at and which
    # contained a mention
    history = set()
    words = sentence.words
    # Scan all subsequences of the sentence
    for start, end in get_all_phrases_in_sentence(sentence, max_mention_length):
        if start in history or end in history:
                continue
        phrase = " ".join([word.word for word in words[start:end]])
        phrase_caseless = phrase.casefold()
        mention = None
        # If the phrase is in the dictionary, then is a mention candidate
        if phrase_caseless in merged_genes_dict:
            mention = Mention("GENE",
                    "|".join(merged_genes_dict[phrase_caseless]), words[start:end])
            add_features(mention, sentence)
            mentions.append(mention)
            # Add to history
            for i in range(start, end + 1):
                history.add(i)
        else: # Potentially generate a random mention
            # Check whether it's a number, we do not want to generate a mention
            # with it.
            is_number = False
            try:
                float(phrase_caseless)
            except:
                pass
            else:
                continue
            has_stop_words = False
            has_verbs = False
            in_merged_dict = False
            for word in words[start:end]:
                if word.word in stopwords_dict:
                    has_stop_words = True
                    break
                if word.pos.startwith("VB"):
                    has_verbs = True
                    break
                # XXX (Matteo) Not perfect. A subset of phrase may be in the
                # dict and we're not checking for this. Low probability, I'd
                # say.
                if word.word in merged_genes_dict:
                    in_merged_dict = True
                    break
            if phrase_caseless.isalnum() and not is_number and not has_stop_words and \
                not has_verbs and not in_merged_dict and \
                len(sentence.words) > 5 and random.random() < RANDOM_EXAMPLES_PROB and \
                random_examples < RANDOM_EXAMPLES_QUOTA:
                # Generate a mention that somewhat resembles what a gene may look like,
                # or at least its role in the sentence.
                # This mention is supervised (as false) in the code calling this function
                mention = Mention("RANDOM", phrase_caseless, sentence[start:end])
                add_features(mention, sentence)
                random_examples += 1
                mentions.append(mention)
    return mentions

# Load the dictionaries that we need
merged_genes_dict = load_dict("merged_genes")
english_dict = load_dict("english")
stopwords_dict = load_dict("stopwords")
pos_mentions_dict = load_dict("pos_gene_mentions")
neg_mentions_dict = load_dict("neg_gene_mentions")
# XXX (Matteo) This dictionaries were used when we didn't have geneRifs to
# label mention candidates as positive. Now they're no longer needed. See also
# comment in supervise().
#nih_grants_dict = load_dict("nih_grants")
#nsf_grants_dict = load_dict("nsf_grants")
#med_acrons_dict = load_dict("med_acrons")
max_mention_length = 0
for key in merged_genes_dict:
    length = len(key.split())
    if length > max_mention_length:
        max_mention_length = length
max_mention_length *= 2 #doubling to take into account commas and who knows what

if __name__ == "__main__":
    # Process the input
    with fileinput.input() as input_files:
        for line in input_files:
            # This is for json
            #line_dict = json.loads(line)
            # This is for tsv
            line_dict = get_dict_from_TSVline(line, ["doc_id", "sent_id", "wordidxs",
            "words", "poses", "ners", "lemmas", "dep_paths", "dep_parents",
            "bounding_boxes", "acronym"], [no_op, int, lambda x :
                TSVstring2list(x, int), TSVstring2list, TSVstring2list,
                TSVstring2list, TSVstring2list, TSVstring2list, lambda x :
                TSVstring2list(x, int), TSVstring2list, no_op])
            sentence = Sentence(line_dict["doc_id"], line_dict["sent_id"],
                    line_dict["wordidxs"], line_dict["words"],
                    line_dict["poses"], line_dict["ners"], line_dict["lemmas"],
                    line_dict["dep_paths"], line_dict["dep_parents"],
                    line_dict["bounding_boxes"])
            # Get list of mentions candidates in this sentence
            mentions = extract(sentence)
            # Add features that use information about other mentions
            if len(mentions) > 1:
                add_features_to_all(mentions, sentence)
            # Supervise according to the mention type
            for mention in mentions:
                if "acronym" in line_dict:
                    if mention.words[0].word == line_dict["acronym"]:
                        mention.type = "ACRONYM"
                        if false_acronyms < ACRONYMS_QUOTA and \
                        random.random() < ACRONYMS_PROB:
                            mention.is_correct = False
                            false_acronyms += 1
                elif mention.type == "RANDOM":
                    # this is a randomly generated example that we assume
                    # to be false
                    mention.is_correct = False
                else:
                    supervise(mention, sentence)
                # Print!
                # This is for json
                #print(mention.json_dump())
                # This is for TSV
                print(mention.tsv_dump())

<|endoftext|>


<commit_before>from django.contrib import admin
from order.models import *
from forms import *
from django.utils.translation import ugettext_lazy as _
import copy

# Register your models here.

class ScientificMissionAdmin(admin.ModelAdmin):
    # Hide this link on the main menu.
    def get_model_perms(self, request):
        perms = super(ScientificMissionAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(ScientificMission, ScientificMissionAdmin)

class SuperOrder(admin.ModelAdmin):
    # Shows the requests according to the user permission.
    # Users defined as superuser or NIRA Admin can see all orders.
    def get_queryset(self, request):
        qs = super(SuperOrder, self).get_queryset(request)
        if request.user.is_nira_admin or request.user.is_superuser:
            return qs
        return qs.filter(requester=request.user)

    # If not superuser or NIRA Admin, the status field becomes read-only.
    def get_readonly_fields(self, request, obj=None):
        ro_fields = super(SuperOrder, self).get_readonly_fields(request, obj)
        if not request.user.is_superuser and not request.user.is_nira_admin:
            ro_fields = list(ro_fields) + ['status']
        return ro_fields

    # If superuser or NIRA Admin, show the requester and protocol fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(SuperOrder, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('requester')
            fieldsets.append((_('Administrative system'), {'fields': ('protocol',)}),)
        return fieldsets

    # If not superuser or NIRA Admin, set the requester as the current user and the status as Open.
    def save_model(self, request, obj, form, change):
        if not request.user.is_superuser and not request.user.is_nira_admin:
            if not change:
                obj.requester = Person.objects.get(user=request.user)
                obj.status = 'o'
        obj.save()

class OrderAdmin(SuperOrder):
    list_display = ('id_order', 'date_modified', 'order_date', 'status', 'type_of_order', 'requester')
    list_per_page = 15
    list_filter = ('status', 'type_of_order', 'requester',)

    # Disable the option to add order.
    def has_add_permission(self, request, obj=None):
        return False

    # Disable the option to delete order.
    def has_delete_permission(self, request, obj=None):
        return False

    # Hide the order link on the main menu.
    def get_model_perms(self, request):
        perms = super(OrderAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(Order, OrderAdmin)

class EventAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Event Info'), {
            'fields': ('name', 'url', 'value', 'start_date', 'end_date', 'invitation')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'name', 'order_date')

    list_display_links = ('order_number', 'status', 'name', 'order_date')

    form = EventAdminForm

admin.site.register(Event, EventAdmin)

class HardwareSoftwareAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Info about the equipment, supplies or miscellaneous'), {
            'fields': ('type', 'quantity', 'url')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = HardwareSoftwareAdminForm

    # If superuser or NIRA admin, show origin and category fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(HardwareSoftwareAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
            fieldsets[0][1]['fields'].append('category')
            fieldsets[0][1]['fields'].append('institution')
        return fieldsets

admin.site.register(HardwareSoftware, HardwareSoftwareAdmin)

class ServiceAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Service Info'), {
            'fields': ('type',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = ServiceAdminForm

    # If superuser or NIRA admin, show origin field.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(ServiceAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
        return fieldsets

admin.site.register(Service, ServiceAdmin)

class TicketAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Ticket Info'), {
            'fields': ('type_transportation', 'type', 'origin', 'destination', 'outbound_date',
                       'outbound_date_preference', 'inbound_date', 'inbound_date_preference', 'note')
        }),
        (_('Purpose of the trip'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'outbound_date', 'inbound_date',
                    'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'outbound_date', 'inbound_date',
                          'order_date')

    form = TicketAdminForm

admin.site.register(Ticket, TicketAdmin)

class DailyStipendAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Daily stipend Info'), {
            'fields': ('origin', 'destination', 'departure', 'arrival')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'order_date')

    # If superuser or NIRA admin, show receiver and mission fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(DailyStipendAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('receiver')
            fieldsets[0][1]['fields'].append('mission')
            fieldsets[0][1]['fields'].append('project_activity')
        return fieldsets

admin.site.register(DailyStipend, DailyStipendAdmin)

class ReimbursementAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Reimbursement Info'), {
            'fields': ('why',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'why', 'order_date')

    list_display_links = ('order_number', 'status', 'why', 'order_date')

admin.site.register(Reimbursement, ReimbursementAdmin)<commit_msg>Bug fix<commit_after>from django.contrib import admin
from order.models import *
from forms import *
from django.utils.translation import ugettext_lazy as _
import copy

# Register your models here.

class ScientificMissionAdmin(admin.ModelAdmin):
    # Hide this link on the main menu.
    def get_model_perms(self, request):
        perms = super(ScientificMissionAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(ScientificMission, ScientificMissionAdmin)

class SuperOrder(admin.ModelAdmin):
    # Shows the requests according to the user permission.
    # Users defined as superuser or NIRA Admin can see all orders.
    def get_queryset(self, request):
        qs = super(SuperOrder, self).get_queryset(request)
        if request.user.is_nira_admin or request.user.is_superuser:
            return qs
        return qs.filter(requester__user=request.user)

    # If not superuser or NIRA Admin, the status field becomes read-only.
    def get_readonly_fields(self, request, obj=None):
        ro_fields = super(SuperOrder, self).get_readonly_fields(request, obj)
        if not request.user.is_superuser and not request.user.is_nira_admin:
            ro_fields = list(ro_fields) + ['status']
        return ro_fields

    # If superuser or NIRA Admin, show the requester and protocol fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(SuperOrder, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('requester')
            fieldsets.append((_('Administrative system'), {'fields': ('protocol',)}),)
        return fieldsets

    # If not superuser or NIRA Admin, set the requester as the current user and the status as Open.
    def save_model(self, request, obj, form, change):
        if not request.user.is_superuser and not request.user.is_nira_admin:
            if not change:
                obj.requester = Person.objects.get(user=request.user)
                obj.status = 'o'
        obj.save()

class OrderAdmin(SuperOrder):
    list_display = ('id_order', 'date_modified', 'order_date', 'status', 'type_of_order', 'requester')
    list_per_page = 15
    list_filter = ('status', 'type_of_order', 'requester',)

    # Disable the option to add order.
    def has_add_permission(self, request, obj=None):
        return False

    # Disable the option to delete order.
    def has_delete_permission(self, request, obj=None):
        return False

    # Hide the order link on the main menu.
    def get_model_perms(self, request):
        perms = super(OrderAdmin, self).get_model_perms(request)
        perms['list_hide'] = True
        return perms

admin.site.register(Order, OrderAdmin)

class EventAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Event Info'), {
            'fields': ('name', 'url', 'value', 'start_date', 'end_date', 'invitation')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'name', 'order_date')

    list_display_links = ('order_number', 'status', 'name', 'order_date')

    form = EventAdminForm

admin.site.register(Event, EventAdmin)

class HardwareSoftwareAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Info about the equipment, supplies or miscellaneous'), {
            'fields': ('type', 'quantity', 'url')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = HardwareSoftwareAdminForm

    # If superuser or NIRA admin, show origin and category fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(HardwareSoftwareAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
            fieldsets[0][1]['fields'].append('category')
            fieldsets[0][1]['fields'].append('institution')
        return fieldsets

admin.site.register(HardwareSoftware, HardwareSoftwareAdmin)

class ServiceAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Service Info'), {
            'fields': ('type',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'type', 'order_date')

    list_display_links = ('order_number', 'status', 'type', 'order_date')

    form = ServiceAdminForm

    # If superuser or NIRA admin, show origin field.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(ServiceAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('origin')
        return fieldsets

admin.site.register(Service, ServiceAdmin)

class TicketAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Ticket Info'), {
            'fields': ('type_transportation', 'type', 'origin', 'destination', 'outbound_date',
                       'outbound_date_preference', 'inbound_date', 'inbound_date_preference', 'note')
        }),
        (_('Purpose of the trip'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'outbound_date', 'inbound_date',
                    'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'outbound_date', 'inbound_date',
                          'order_date')

    form = TicketAdminForm

admin.site.register(Ticket, TicketAdmin)

class DailyStipendAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Daily stipend Info'), {
            'fields': ('origin', 'destination', 'departure', 'arrival')
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'origin', 'destination', 'order_date')

    list_display_links = ('order_number', 'status', 'origin', 'destination', 'order_date')

    # If superuser or NIRA admin, show receiver and mission fields.
    def get_fieldsets(self, request, obj=None):
        fieldsets = copy.deepcopy(super(DailyStipendAdmin, self).get_fieldsets(request, obj))
        if request.user.is_nira_admin or request.user.is_superuser:
            fieldsets[0][1]['fields'].append('receiver')
            fieldsets[0][1]['fields'].append('mission')
            fieldsets[0][1]['fields'].append('project_activity')
        return fieldsets

admin.site.register(DailyStipend, DailyStipendAdmin)

class ReimbursementAdmin(SuperOrder):
    fieldsets = [
        (None, {
            'fields': ['status']
        }),
        (_('Reimbursement Info'), {
            'fields': ('why',)
        }),
        (_('Purpose'), {
            'fields': ('justification',)
        }),
    ]

    list_display = ('order_number', 'status', 'requester', 'why', 'order_date')

    list_display_links = ('order_number', 'status', 'why', 'order_date')

admin.site.register(Reimbursement, ReimbursementAdmin)<|endoftext|>