<commit_before>import claripy
import nose
import ctypes

from archinfo import Endness
from angr import SimState

def test_simple_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    def check_read(val):
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr, 8, endness=Endness.LE), cast_to=int), val)

        nose.tools.assert_equal(s.mem[addr].char.concrete, chr(val & 0xFF))
        nose.tools.assert_equal(s.mem[addr].byte.concrete, val & 0xFF)

        nose.tools.assert_equal(s.mem[addr].int16_t.concrete, ctypes.c_int16(val & 0xFFFF).value)
        nose.tools.assert_equal(s.mem[addr].uint16_t.concrete, val & 0xFFFF)

        nose.tools.assert_equal(s.mem[addr].qword.concrete, val)

    s.memory.store(addr, claripy.BVV(0x11223344aabbcc7d, 64), endness=Endness.LE)
    check_read(0x11223344aabbcc7d)

    # test storing
    s.mem[addr].uint16_t = 0xef6d
    check_read(0x11223344aabbef6d)

def test_string_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    def check_read(val):
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr, len(val)), cast_to=str), val)
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr + len(val), 1), cast_to=int), 0)

        nose.tools.assert_equal(s.mem[addr].string.concrete, val)

    s.memory.store(addr, "a string!\0")
    check_read("a string!")

    # not supported yet
    # s.mem[addr].string = "shorter"
    # check_read("shorter")

    # s.mem[addr].string = "a longer string"
    # check_read("a longer string")

def test_array_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    s.memory.store(addr, claripy.BVV(0x1, 32), endness=Endness.LE)
    s.memory.store(addr + 4, claripy.BVV(0x2, 32), endness=Endness.LE)
    s.memory.store(addr + 8, claripy.BVV(0x3, 32), endness=Endness.LE)
    s.memory.store(addr + 12, claripy.BVV(0x4, 32), endness=Endness.LE)
    s.memory.store(addr + 16, claripy.BVV(0x5, 32), endness=Endness.LE)

    nose.tools.assert_equal(s.mem[addr].dword.array(5).concrete, [0x1, 0x2, 0x3, 0x4, 0x5])
    nose.tools.assert_equal(s.mem[addr].dword.array(5)[2].concrete, 0x3)
    nose.tools.assert_equal(s.mem[addr].qword.array(2).concrete, [0x0000000200000001, 0x0000000400000003])
    nose.tools.assert_equal(s.mem[addr].dword.array(2).array(2).concrete, [[0x1, 0x2], [0x3, 0x4]])

    s.mem[addr].dword.array(5)[3] = 10
    nose.tools.assert_equals(s.se.eval(s.memory.load(addr + 12, 4, endness=Endness.LE), cast_to=int), 10)

    s.mem[addr].dword.array(5).store([20,2,3,4,5])
    nose.tools.assert_equals(s.mem[addr].dword.array(4).concrete, [20,2,3,4])

    s.mem[addr].dword.array(2).array(2).store([[1,2], [4,3]])
    nose.tools.assert_equals(s.mem[addr].dword.array(4).concrete, [1,2,4,3])

def test_pointer_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0
    ptraddr = 0xcd0

    s.memory.store(ptraddr, claripy.BVV(addr, 64), endness=Endness.LE)
    s.memory.store(addr, "abcdef\0")

    nose.tools.assert_equal(s.mem[ptraddr].deref.string.concrete, "abcdef")
    s.mem[ptraddr].deref.dword = 123954
    nose.tools.assert_equal(s.se.eval(s.memory.load(addr, 4, endness=Endness.LE), cast_to=int), 123954)
    nose.tools.assert_equal(s.mem[ptraddr].deref.dword.concrete, 123954)

if __name__ == '__main__':
    test_simple_concrete()
    test_string_concrete()
    test_array_concrete()
    test_pointer_concrete()
<commit_msg>Add test for structs in memory view<commit_after>import claripy
import nose
import ctypes

from archinfo import Endness
from angr import SimState
from angr.sim_type import register_types, parse_types

def test_simple_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    def check_read(val):
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr, 8, endness=Endness.LE), cast_to=int), val)

        nose.tools.assert_equal(s.mem[addr].char.concrete, chr(val & 0xFF))
        nose.tools.assert_equal(s.mem[addr].byte.concrete, val & 0xFF)

        nose.tools.assert_equal(s.mem[addr].int16_t.concrete, ctypes.c_int16(val & 0xFFFF).value)
        nose.tools.assert_equal(s.mem[addr].uint16_t.concrete, val & 0xFFFF)

        nose.tools.assert_equal(s.mem[addr].qword.concrete, val)

    s.memory.store(addr, claripy.BVV(0x11223344aabbcc7d, 64), endness=Endness.LE)
    check_read(0x11223344aabbcc7d)

    # test storing
    s.mem[addr].uint16_t = 0xef6d
    check_read(0x11223344aabbef6d)

def test_string_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    def check_read(val):
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr, len(val)), cast_to=str), val)
        nose.tools.assert_equal(s.se.eval(s.memory.load(addr + len(val), 1), cast_to=int), 0)

        nose.tools.assert_equal(s.mem[addr].string.concrete, val)

    s.memory.store(addr, "a string!\0")
    check_read("a string!")

    # not supported yet
    # s.mem[addr].string = "shorter"
    # check_read("shorter")

    # s.mem[addr].string = "a longer string"
    # check_read("a longer string")

def test_array_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0

    s.memory.store(addr, claripy.BVV(0x1, 32), endness=Endness.LE)
    s.memory.store(addr + 4, claripy.BVV(0x2, 32), endness=Endness.LE)
    s.memory.store(addr + 8, claripy.BVV(0x3, 32), endness=Endness.LE)
    s.memory.store(addr + 12, claripy.BVV(0x4, 32), endness=Endness.LE)
    s.memory.store(addr + 16, claripy.BVV(0x5, 32), endness=Endness.LE)

    nose.tools.assert_equal(s.mem[addr].dword.array(5).concrete, [0x1, 0x2, 0x3, 0x4, 0x5])
    nose.tools.assert_equal(s.mem[addr].dword.array(5)[2].concrete, 0x3)
    nose.tools.assert_equal(s.mem[addr].qword.array(2).concrete, [0x0000000200000001, 0x0000000400000003])
    nose.tools.assert_equal(s.mem[addr].dword.array(2).array(2).concrete, [[0x1, 0x2], [0x3, 0x4]])

    s.mem[addr].dword.array(5)[3] = 10
    nose.tools.assert_equals(s.se.eval(s.memory.load(addr + 12, 4, endness=Endness.LE), cast_to=int), 10)

    s.mem[addr].dword.array(5).store([20,2,3,4,5])
    nose.tools.assert_equals(s.mem[addr].dword.array(4).concrete, [20,2,3,4])

    s.mem[addr].dword.array(2).array(2).store([[1,2], [4,3]])
    nose.tools.assert_equals(s.mem[addr].dword.array(4).concrete, [1,2,4,3])

def test_pointer_concrete():
    s = SimState(arch="AMD64")
    addr = 0xba5e0
    ptraddr = 0xcd0

    s.memory.store(ptraddr, claripy.BVV(addr, 64), endness=Endness.LE)
    s.memory.store(addr, "abcdef\0")

    nose.tools.assert_equal(s.mem[ptraddr].deref.string.concrete, "abcdef")
    s.mem[ptraddr].deref.dword = 123954
    nose.tools.assert_equal(s.se.eval(s.memory.load(addr, 4, endness=Endness.LE), cast_to=int), 123954)
    nose.tools.assert_equal(s.mem[ptraddr].deref.dword.concrete, 123954)

def test_structs():
    s = SimState(arch='AMD64')

    register_types(parse_types("""
struct abcd {
  int a;
  long b;
};
"""))

    s.mem[0x8000].struct.abcd = {'a': 10, 'b': 20}
    assert s.mem[0x8000].struct.abcd.a.concrete == 10
    assert s.solver.eval(s.memory.load(0x8000, 16), cast_to=str) == '0a000000000000001400000000000000'.decode('hex')

if __name__ == '__main__':
    test_simple_concrete()
    test_string_concrete()
    test_array_concrete()
    test_pointer_concrete()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_messier
----------------------------------

Tests for `messier` module.
"""

import unittest
import os
import tempfile
import yaml
import hashlib

from messier import messier
from messier.serverspec_handler import cd

from messier.exceptions import VagrantfileNotFound

class TestMessier(unittest.TestCase):

    def setUp(self):
        self.messier = messier.Messier()

    def tearDown(self):
        pass

    def test_000_something(self):
        pass

    @unittest.skip("Not sure whether missing Vagrantfile should throw an error.")
    def test_empty_config(self):
        """
        Create a Messier object with no config and ensure config is empty.
        """
        temp_dir = tempfile.mkdtemp()
        with cd(temp_dir):
            m = messier.Messier()
        self.assertEqual(m.config, {})

    def test_empty_config_raises_exception(self):
        """
        Create a Messier object with no config and ensure config is empty.
        """
        temp_dir = tempfile.mkdtemp()
        with self.assertRaises(VagrantfileNotFound):
            with cd(temp_dir):
                m = messier.Messier()

    def test_custom_config(self, content=None):
        """
        Create a Messier object with a custom config file and validate the config matches.
        """
        desired_boxes = ['ubuntu/trusty64', 'ubuntu/vivid64', 'debian/jessie64']
        content = dict(vagrant_boxes=desired_boxes)
        config_path = tempfile.mktemp()
        with open(config_path, 'w') as f:
            f.write(yaml.dump(content, default_flow_style=True))
        m = messier.Messier(config_file=config_path)
        assert os.path.exists(config_path)
        for box in desired_boxes:
            assert box in m.config['vagrant_boxes']

    def test_list_vms(self, content=None):
        """
        Ensure named VM in Vagrantfile is returned by `messier list`.
        """
        # Uses tests directory?
        m = messier.Messier()
        for vm in ("client", "server"):
          assert vm in m.vms

    @unittest.skip("`init` command not yet supported")
    def test_init_creates_vagrantfile_if_none(self):
        """
        Create a new Messier project with a Vagrantfile.
        """
        temp_dir = tempfile.mkdtemp()
        assert not os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        with cd(temp_dir):
            m = messier.Messier()
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))

    @unittest.skip("`init` command not yet supported")
    def test_init_does_not_clobber_vagrantfile(self):
        """
        Create a new Messier project and refuse to overwrite preexisting Vagrantfile.
        """
        temp_dir = tempfile.mkdtemp()
        vagrantfile = tempfile.mktemp(dir=temp_dir)
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        original_checksum = hashlib.sha256(vagrantfile).hexdigest()
        with cd(temp_dir):
            m = messier.Messier()
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        new_checksum = hashlib.sha256(vagrantfile).hexdigest()
        assert new_checksum == original_checksum

if __name__ == '__main__':
    import sys
    sys.exit(unittest.main())
<commit_msg>FIX: proper assertion for vms<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_messier
----------------------------------

Tests for `messier` module.
"""

import unittest
import os
import tempfile
import yaml
import hashlib

from messier import messier
from messier.serverspec_handler import cd

from messier.exceptions import VagrantfileNotFound

class TestMessier(unittest.TestCase):

    def setUp(self):
        self.messier = messier.Messier()

    def tearDown(self):
        pass

    def test_000_something(self):
        pass

    @unittest.skip("Not sure whether missing Vagrantfile should throw an error.")
    def test_empty_config(self):
        """
        Create a Messier object with no config and ensure config is empty.
        """
        temp_dir = tempfile.mkdtemp()
        with cd(temp_dir):
            m = messier.Messier()
        self.assertEqual(m.config, {})

    def test_empty_config_raises_exception(self):
        """
        Create a Messier object with no config and ensure config is empty.
        """
        temp_dir = tempfile.mkdtemp()
        with self.assertRaises(VagrantfileNotFound):
            with cd(temp_dir):
                m = messier.Messier()

    def test_custom_config(self, content=None):
        """
        Create a Messier object with a custom config file and validate the config matches.
        """
        desired_boxes = ['ubuntu/trusty64', 'ubuntu/vivid64', 'debian/jessie64']
        content = dict(vagrant_boxes=desired_boxes)
        config_path = tempfile.mktemp()
        with open(config_path, 'w') as f:
            f.write(yaml.dump(content, default_flow_style=True))
        m = messier.Messier(config_file=config_path)
        assert os.path.exists(config_path)
        for box in desired_boxes:
            assert box in m.config['vagrant_boxes']

    def test_list_vms(self, content=None):
        """
        Ensure named VM in Vagrantfile is returned by `messier list`.
        """
        # Uses tests directory?
        with cd(os.path.abspath(os.path.curdir)):
            m = messier.Messier()
            for vm in ("client", "server"):
              assert vm in [vm.name for vm in m.vms]

    @unittest.skip("`init` command not yet supported")
    def test_init_creates_vagrantfile_if_none(self):
        """
        Create a new Messier project with a Vagrantfile.
        """
        temp_dir = tempfile.mkdtemp()
        assert not os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        with cd(temp_dir):
            m = messier.Messier()
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))

    @unittest.skip("`init` command not yet supported")
    def test_init_does_not_clobber_vagrantfile(self):
        """
        Create a new Messier project and refuse to overwrite preexisting Vagrantfile.
        """
        temp_dir = tempfile.mkdtemp()
        vagrantfile = tempfile.mktemp(dir=temp_dir)
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        original_checksum = hashlib.sha256(vagrantfile).hexdigest()
        with cd(temp_dir):
            m = messier.Messier()
        assert os.path.exists(os.path.join(temp_dir, 'Vagrantfile'))
        new_checksum = hashlib.sha256(vagrantfile).hexdigest()
        assert new_checksum == original_checksum

if __name__ == '__main__':
    import sys
    sys.exit(unittest.main())
<|endoftext|>


<commit_before>from django.db.models import Q
from reviewboard.extensions.base import Extension
from reviewboard.extensions.hooks import TemplateHook
from reviewboard.urls import reviewable_url_names, review_request_url_names
from reviewboard.webapi.resources.review_group import ReviewGroupResource

apply_to_url_names = set(reviewable_url_names + review_request_url_names)

class AutocompleteContainsReviewGroupResource(ReviewGroupResource):
    def get_queryset(self, request, *args, **kwargs):
        # Create mutable QueryDict and pop the 'q' parameter
        request.GET = request.GET.copy()
        try:
            search_q = request.GET.pop('q')[0]
        except:
            search_q = None

        query = ReviewGroupResource.get_queryset(
            self, request=request, *args, **kwargs)

        if search_q:
            q = Q(name__icontains=search_q)

            if request.GET.get('displayname', None):
                q = q | Q(display_name__icontains=search_q)

            query = query.filter(q)

        return query

review_group_resource = AutocompleteContainsReviewGroupResource()

class AutocompleteContains(Extension):
    metadata = {
        'Name': 'Autocomplete Contains',
        'Summary': 'Enhance review groups autocomplete',
        'Author': 'Erik Johansson',
        'Author-email': 'erik@ejohansson.se',
        'License': 'MIT',
        'Home-page': 'https://github.com/erijo/rb-autocomplete-contains'
    }

    resources = [review_group_resource]

    def initialize(self):
        TemplateHook(self, 'base-scripts-post',
                     'autocomplete_contains/modify_field.html',
                     apply_to=apply_to_url_names)
<commit_msg>Use last element in list returned from QueryDict.pop<commit_after>from django.db.models import Q
from reviewboard.extensions.base import Extension
from reviewboard.extensions.hooks import TemplateHook
from reviewboard.urls import reviewable_url_names, review_request_url_names
from reviewboard.webapi.resources.review_group import ReviewGroupResource

apply_to_url_names = set(reviewable_url_names + review_request_url_names)

class AutocompleteContainsReviewGroupResource(ReviewGroupResource):
    def get_queryset(self, request, *args, **kwargs):
        # Create mutable QueryDict and pop the 'q' parameter
        request.GET = request.GET.copy()
        try:
            search_q = request.GET.pop('q')[-1]
        except:
            search_q = None

        query = ReviewGroupResource.get_queryset(
            self, request=request, *args, **kwargs)

        if search_q:
            q = Q(name__icontains=search_q)

            if request.GET.get('displayname', None):
                q = q | Q(display_name__icontains=search_q)

            query = query.filter(q)

        return query

review_group_resource = AutocompleteContainsReviewGroupResource()

class AutocompleteContains(Extension):
    metadata = {
        'Name': 'Autocomplete Contains',
        'Summary': 'Enhance review groups autocomplete',
        'Author': 'Erik Johansson',
        'Author-email': 'erik@ejohansson.se',
        'License': 'MIT',
        'Home-page': 'https://github.com/erijo/rb-autocomplete-contains'
    }

    resources = [review_group_resource]

    def initialize(self):
        TemplateHook(self, 'base-scripts-post',
                     'autocomplete_contains/modify_field.html',
                     apply_to=apply_to_url_names)
<|endoftext|>


<commit_before>"""
Evaluation test is performed for the following dataset.
https://www.clips.uantwerpen.be/conll2000/chunking/output.html
"""
import os
import random
import subprocess
import unittest

from seqeval.metrics import f1_score, accuracy_score, classification_report, precision_score, recall_score
from seqeval.metrics.sequence_labeling import get_entities

class TestMetrics(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.file_name = os.path.join(os.path.dirname(__file__), 'data/ground_truth.txt')
        cls.y_true, cls.y_pred = cls.load_labels(cls, cls.file_name)

    def test_get_entities(self):
        y_true = ['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O', 'B-PER', 'I-PER']
        self.assertEqual(get_entities(y_true), [('MISC', 3, 5), ('PER', 7, 8)])

    def test_get_entities_with_suffix_style(self):
        y_true = ['O', 'O', 'O', 'MISC-B', 'MISC-I', 'MISC-I', 'O', 'PER-B', 'PER-I']
        self.assertEqual(get_entities(y_true, suffix=True), [('MISC', 3, 5), ('PER', 7, 8)])

    def test_classification_report(self):
        print(classification_report(self.y_true, self.y_pred))

    def test_by_ground_truth(self):
        with open(self.file_name) as f:
            output = subprocess.check_output(['perl', 'conlleval.pl'], stdin=f).decode('utf-8')
            acc_true, p_true, r_true, f1_true = self.parse_conlleval_output(output)

            acc_pred = accuracy_score(self.y_true, self.y_pred)
            p_pred = precision_score(self.y_true, self.y_pred)
            r_pred = recall_score(self.y_true, self.y_pred)
            f1_pred = f1_score(self.y_true, self.y_pred)

            self.assertLess(abs(acc_pred - acc_true), 1e-4)
            self.assertLess(abs(p_pred - p_true), 1e-4)
            self.assertLess(abs(r_pred - r_true), 1e-4)
            self.assertLess(abs(f1_pred - f1_true), 1e-4)

    def test_statistical_tests(self):
        filepath = 'eval_data.txt'
        for prefix in ['BIO', 'EIO']:
            for i in range(10000):
                print('Iteration: {}'.format(i))
                self.generate_eval_data(filepath, prefix)
                y_true, y_pred = self.load_labels(filepath)
                with open(filepath) as f:
                    output = subprocess.check_output(['perl', 'conlleval.pl'], stdin=f).decode('utf-8')
                    acc_true, p_true, r_true, f1_true = self.parse_conlleval_output(output)

                    acc_pred = accuracy_score(y_true, y_pred)
                    p_pred = precision_score(y_true, y_pred)
                    r_pred = recall_score(y_true, y_pred)
                    f1_pred = f1_score(y_true, y_pred)

                    self.assertLess(abs(acc_pred - acc_true), 1e-4)
                    self.assertLess(abs(p_pred - p_true), 1e-4)
                    self.assertLess(abs(r_pred - r_true), 1e-4)
                    self.assertLess(abs(f1_pred - f1_true), 1e-4)

        os.remove(filepath)

    def load_labels(self, filename):
        y_true, y_pred = [], []
        with open(filename) as f:
            tags_true, tags_pred = [], []
            for line in f:
                line = line.rstrip()
                if len(line) == 0:
                    if len(tags_true) != 0:
                        y_true.append(tags_true)
                        y_pred.append(tags_pred)
                        tags_true, tags_pred = [], []
                else:
                    _, _, tag_true, tag_pred = line.split(' ')
                    tags_true.append(tag_true)
                    tags_pred.append(tag_pred)
            else:
                y_true.append(tags_true)
                y_pred.append(tags_pred)
        return y_true, y_pred

    @staticmethod
    def parse_conlleval_output(text):
        eval_line = text.split('\n')[1]
        items = eval_line.split(' ')
        accuracy, precision, recall = [item[:-2] for item in items if '%' in item]
        f1 = items[-1]

        accuracy = float(accuracy) / 100
        precision = float(precision) / 100
        recall = float(recall) / 100
        f1 = float(f1) / 100

        return accuracy, precision, recall, f1

    @staticmethod
    def generate_eval_data(filepath, prefixes='BIO'):
        types = ['PER', 'MISC', 'ORG', 'LOC']
        report = ''
        raw_fmt = '{} {} {} {}\n'
        for i in range(1000):
            type_true = random.choice(types)
            type_pred = random.choice(types)
            prefix_true = random.choice(prefixes)
            prefix_pred = random.choice(prefixes)
            true_out = 'O' if prefix_true == 'O' else '{}-{}'.format(prefix_true, type_true)
            pred_out = 'O' if prefix_pred == 'O' else '{}-{}'.format(prefix_pred, type_pred)
            report += raw_fmt.format('X', 'X', true_out, pred_out)

            # end of sentence
            if random.random() > 0.95:
                report += '\n'

        with open(filepath, 'w') as f:
            f.write(report)
<commit_msg>add tests for the evaluation of suffix-style dataset<commit_after>"""
Evaluation test is performed for the following dataset.
https://www.clips.uantwerpen.be/conll2000/chunking/output.html
"""
import os
import random
import subprocess
import unittest

from seqeval.metrics import f1_score, accuracy_score, classification_report, precision_score, recall_score
from seqeval.metrics.sequence_labeling import get_entities

class TestMetrics(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        cls.file_name = os.path.join(os.path.dirname(__file__), 'data/ground_truth.txt')
        cls.y_true, cls.y_pred = cls.load_labels(cls, cls.file_name)
        cls.inv_file_name = os.path.join(os.path.dirname(__file__), 'data/ground_truth_inv.txt')
        cls.y_true_inv, cls.y_pred_inv = cls.load_labels(cls, cls.inv_file_name)

    def test_get_entities(self):
        y_true = ['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O', 'B-PER', 'I-PER']
        self.assertEqual(get_entities(y_true), [('MISC', 3, 5), ('PER', 7, 8)])

    def test_get_entities_with_suffix_style(self):
        y_true = ['O', 'O', 'O', 'MISC-B', 'MISC-I', 'MISC-I', 'O', 'PER-B', 'PER-I']
        self.assertEqual(get_entities(y_true, suffix=True), [('MISC', 3, 5), ('PER', 7, 8)])

    def test_classification_report(self):
        print(classification_report(self.y_true, self.y_pred))

    def test_inv_classification_report(self):
        print(classification_report(self.y_true_inv, self.y_pred_inv, suffix=True))

    def test_by_ground_truth(self):
        with open(self.file_name) as f:
            output = subprocess.check_output(['perl', 'conlleval.pl'], stdin=f).decode('utf-8')
            acc_true, p_true, r_true, f1_true = self.parse_conlleval_output(output)

            acc_pred = accuracy_score(self.y_true, self.y_pred)
            p_pred = precision_score(self.y_true, self.y_pred)
            r_pred = recall_score(self.y_true, self.y_pred)
            f1_pred = f1_score(self.y_true, self.y_pred)

            self.assertLess(abs(acc_pred - acc_true), 1e-4)
            self.assertLess(abs(p_pred - p_true), 1e-4)
            self.assertLess(abs(r_pred - r_true), 1e-4)
            self.assertLess(abs(f1_pred - f1_true), 1e-4)

    def test_metrics_for_inv_data(self):
        with open(self.file_name) as f:
            acc_pred = accuracy_score(self.y_true, self.y_pred)
            p_pred = precision_score(self.y_true, self.y_pred)
            r_pred = recall_score(self.y_true, self.y_pred)
            f1_pred = f1_score(self.y_true, self.y_pred)

            acc_pred_inv = accuracy_score(self.y_true_inv, self.y_pred_inv)
            p_pred_inv = precision_score(self.y_true_inv, self.y_pred_inv, suffix=True)
            r_pred_inv = recall_score(self.y_true_inv, self.y_pred_inv, suffix=True)
            f1_pred_inv = f1_score(self.y_true_inv, self.y_pred_inv, suffix=True)

            self.assertLess(abs(acc_pred - acc_pred_inv), 1e-4)
            self.assertLess(abs(p_pred - p_pred_inv), 1e-4)
            self.assertLess(abs(r_pred - r_pred_inv), 1e-4)
            self.assertLess(abs(f1_pred - f1_pred_inv), 1e-4)

    def test_statistical_tests(self):
        filepath = 'eval_data.txt'
        for prefix in ['BIO', 'EIO']:
            for i in range(10000):
                print('Iteration: {}'.format(i))
                self.generate_eval_data(filepath, prefix)
                y_true, y_pred = self.load_labels(filepath)
                with open(filepath) as f:
                    output = subprocess.check_output(['perl', 'conlleval.pl'], stdin=f).decode('utf-8')
                    acc_true, p_true, r_true, f1_true = self.parse_conlleval_output(output)

                    acc_pred = accuracy_score(y_true, y_pred)
                    p_pred = precision_score(y_true, y_pred)
                    r_pred = recall_score(y_true, y_pred)
                    f1_pred = f1_score(y_true, y_pred)

                    self.assertLess(abs(acc_pred - acc_true), 1e-4)
                    self.assertLess(abs(p_pred - p_true), 1e-4)
                    self.assertLess(abs(r_pred - r_true), 1e-4)
                    self.assertLess(abs(f1_pred - f1_true), 1e-4)

        os.remove(filepath)

    def load_labels(self, filename):
        y_true, y_pred = [], []
        with open(filename) as f:
            tags_true, tags_pred = [], []
            for line in f:
                line = line.rstrip()
                if len(line) == 0:
                    if len(tags_true) != 0:
                        y_true.append(tags_true)
                        y_pred.append(tags_pred)
                        tags_true, tags_pred = [], []
                else:
                    _, _, tag_true, tag_pred = line.split(' ')
                    tags_true.append(tag_true)
                    tags_pred.append(tag_pred)
            else:
                y_true.append(tags_true)
                y_pred.append(tags_pred)
        return y_true, y_pred

    @staticmethod
    def parse_conlleval_output(text):
        eval_line = text.split('\n')[1]
        items = eval_line.split(' ')
        accuracy, precision, recall = [item[:-2] for item in items if '%' in item]
        f1 = items[-1]

        accuracy = float(accuracy) / 100
        precision = float(precision) / 100
        recall = float(recall) / 100
        f1 = float(f1) / 100

        return accuracy, precision, recall, f1

    @staticmethod
    def generate_eval_data(filepath, prefixes='BIO'):
        types = ['PER', 'MISC', 'ORG', 'LOC']
        report = ''
        raw_fmt = '{} {} {} {}\n'
        for i in range(1000):
            type_true = random.choice(types)
            type_pred = random.choice(types)
            prefix_true = random.choice(prefixes)
            prefix_pred = random.choice(prefixes)
            true_out = 'O' if prefix_true == 'O' else '{}-{}'.format(prefix_true, type_true)
            pred_out = 'O' if prefix_pred == 'O' else '{}-{}'.format(prefix_pred, type_pred)
            report += raw_fmt.format('X', 'X', true_out, pred_out)

            # end of sentence
            if random.random() > 0.95:
                report += '\n'

        with open(filepath, 'w') as f:
            f.write(report)
<|endoftext|>


<commit_before>"""
QueryRunner is the function that the workers use, to execute queries. This is the Redshift
(PostgreSQL in fact) version, but easily we can write another to support additional databases
(MySQL and others).

Because the worker just pass the query, this can be used with any data store that has some sort of
query language (for example: HiveQL).
"""
import logging
import json
import MySQLdb
import sys
import select
from .utils import JSONEncoder

def mysql(connection_string):
    def column_friendly_name(column_name):
        return column_name

    def query_runner(query):
        # # TODO: remove these lines
        # import settings
        # connection_string = settings.CONNECTION_STRING
        # # ENDTODO
        print connection_string

        if connection_string.endswith(';'):
            connection_string = connection_string[0:-1]

        connections_params = [entry.split('=')[1] for entry in connection_string.split(';')]
        connection = MySQLdb.connect(*connections_params)
        cursor = connection.cursor()

        logging.debug("mysql got query: %s", query)

        try:
            cursor.execute(query)

            data = cursor.fetchall()

            num_fields = len(cursor.description)
            column_names = [i[0] for i in cursor.description]

            rows = [dict(zip(column_names, row)) for row in data]

            columns = [{'name': col_name,
                        'friendly_name': column_friendly_name(col_name),
                        'type': None} for col_name in column_names]

            data = {'columns': columns, 'rows': rows}
            json_data = json.dumps(data, cls=JSONEncoder)
            error = None
            cursor.close()
        except MySQLdb.Error, e:
            json_data = None
            error = e.message
        except Exception as e:
            raise sys.exc_info()[1], None, sys.exc_info()[2]
        finally:
            connection.close()

        return json_data, error

    return query_runner<commit_msg>Fixed MySQL Runner<commit_after>"""
QueryRunner is the function that the workers use, to execute queries. This is the Redshift
(PostgreSQL in fact) version, but easily we can write another to support additional databases
(MySQL and others).

Because the worker just pass the query, this can be used with any data store that has some sort of
query language (for example: HiveQL).
"""
import logging
import json
import MySQLdb
import sys
import select
from .utils import JSONEncoder

def mysql(connection_string):
    if connection_string.endswith(';'):
        connection_string = connection_string[0:-1]

    def query_runner(query):
        connections_params = [entry.split('=')[1] for entry in connection_string.split(';')]
        connection = MySQLdb.connect(*connections_params)
        cursor = connection.cursor()

        logging.debug("mysql got query: %s", query)

        try:
            cursor.execute(query)

            data = cursor.fetchall()

            num_fields = len(cursor.description)
            column_names = [i[0] for i in cursor.description]

            rows = [dict(zip(column_names, row)) for row in data]

            columns = [{'name': col_name,
                        'friendly_name': col_name,
                        'type': None} for col_name in column_names]

            data = {'columns': columns, 'rows': rows}
            json_data = json.dumps(data, cls=JSONEncoder)
            error = None
            cursor.close()
        except MySQLdb.Error, e:
            json_data = None
            error = e.message
        except Exception as e:
            raise sys.exc_info()[1], None, sys.exc_info()[2]
        finally:
            connection.close()

        return json_data, error

    return query_runner<|endoftext|>


<commit_before># =========================================================================
# Copyright 2015 Yunify, Inc.
# -------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

import json

from .key import Key
from .multipart import MultiPartUpload
from .exception import get_response_error

class Bucket(object):

    DefaultContentType = "application/oct-stream"

    def __init__(self, connection=None, name=None):
        self.connection = connection
        self.name = name

    def __repr__(self):
        return '<Bucket: %s>' % self.name

    def __contains__(self, key_name):
        return not (self.get_key(key_name) is None)

    def __getitem__(self, key_name):
        key = self.get_key(key_name)
        if key:
            return key
        else:
            raise KeyError

    def __len__(self):
        pass

    def get_key(self, key_name, validate=True):
        """ Retrieves an object by name.
        Returns: An instance of a Key object or None

        Keyword arguments:
        key_name - The name of the bucket
        validate - If True, the function will try to verify the object exists
            on the service-side (Default: True)
        """

        if not validate:
            return Key(self, key_name)

        response = self.connection.make_request("HEAD", self.name, key_name)

        if response.status == 200:
            return Key(self, key_name)
        elif response.status == 401:
            err = get_response_error(response)
            err.code = "invalid_access_key_id"
            err.message = "Request not authenticated, Access Key ID is either " \
                "missing or invalid."
            raise err
        elif response.status == 403:
            err = get_response_error(response)
            err.code = "permission_denied"
            err.message = "You don't have enough permission to accomplish " \
                "this request."
            raise err
        elif response.status == 404:
            err = get_response_error(response)
            err.code = "object_not_exists"
            err.message = "The object you are accessing doesn't exist."
            raise err
        else:
            err = get_response_error(response)
            raise err

    def new_key(self, key_name):
        """ Create a new object within the bucket.
        The function will not PUT an object to service-side, utils function
        send_file of the return instance is called.
        Returns: An instance of a Key object

        Keyword arguments:
        key_name - The name of the object
        """
        return Key(self, key_name)

    def delete_key(self, key_name):
        """ Deleted the particular object by object name.

        Keyword arguments:
        key_name - The name of the object
        """
        response = self.connection.make_request("DELETE", self.name, key_name)
        if response.status == 204:
            return True
        else:
            err = get_response_error(response)
            raise err

    def list(self, prefix=None, delimiter=None, marker=None, limit=None):
        """ List objects of the bucket.

        Keyword arguments:
        prefix - Limits the response to keys that begin with the specified prefix
        delimiter - A character you use to group keys
        marker - Specifies the key to start with when listing objects
        limit - The count of objects that the request returns
        """
        params = {}
        if prefix:
            params["prefix"] = prefix
        if delimiter:
            params["delimiter"] = delimiter
        if marker:
            params["marker"] = marker
        if limit:
            params["limit"] = str(limit)
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            result_set = []
            for k in resp["keys"]:
                key = Key(self, k["key"])
                key.content_type = k["mime_type"]
                result_set.append(key)
            return result_set
        else:
            err = get_response_error(response)
            raise err

    def delete(self):
        """ Delete the bucket
        """
        response = self.connection.make_request(
            "DELETE", self.name, num_retries=6)
        if response.status == 204:
            return True
        else:
            raise get_response_error(response)

    def stats(self):
        """ Retrieve the bucket meta information.
        """
        params = {"stats": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def get_acl(self):
        """ Retrieve the bucket access control list.
        """
        params = {"acl": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def set_acl(self, acls):
        """ Set the bucket access control list.

        Keyword arguments:
        acls - The list of ACL infomation
        """
        params = {"acl": None}
        data = json.dumps(acls)
        response = self.connection.make_request("PUT", self.name,
                                                params=params, data=data)
        if response.status == 200:
            return True
        else:
            err = get_response_error(response)
            raise err

    def get_cors(self):
        """ Retrieve the bucket CORS settings.
        """
        params = {"cors": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def set_cors(self, cors):
        """ Set the bucket CORS settings.

        Keyword arguments:
        cors - a dict object contains CORS rulses
        """
        params = {"cors": None}
        data = json.dumps(cors)
        response = self.connection.make_request("PUT", self.name,
                                                params=params, data=data)
        if response.status == 200:
            return True
        else:
            err = get_response_error(response)
            raise err

    def delete_cors(self, cors):
        """ Delete the bucket CORS settings.
        """
        params = {"cors": None}
        response = self.connection.make_request("DELETE", self.name, params=params)
        if response.status == 200:
            return True
        else:
            err = get_response_error(response)
            raise err

    def initiate_multipart_upload(self, key_name, content_type=None):
        """Initiate a multipart upload.
        Returns: An instance of MultiPartUpload

        Keyword arguments:
        key_name - The object name
        content_type - The content type of the object
        """
        params = {"uploads": None}
        headers = {
            "Content-Type": content_type or self.DefaultContentType
        }
        response = self.connection.make_request(
            "POST", self.name, key_name, headers=headers, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            handler = MultiPartUpload(self, key_name, resp["upload_id"])
            return handler
        else:
            err = get_response_error(response)
            raise err

    def cancel_multipart_upload(self, key_name, upload_id):
        """Abort the multipart upload.

        Keyword arguments:
        key_name - The object name
        upload_id - ID for the initiated multipart upload
        """
        params = {
            "upload_id": upload_id,
        }
        response = self.connection.make_request(
            "DELETE", self.name, key_name, params=params)
        if response.status == 204:
            return True
        else:
            err = get_response_error(response)
            raise err

    def complete_multipart_upload(self, key_name, upload_id, parts):
        """Complete the multipart upload.

        Keyword arguments:
        key_name - The object name
        upload_id - ID for the initiated multipart upload
        parts - The list of the multiparts which need to be combined
        """
        params = {
            "upload_id": upload_id,
        }
        object_parts = []
        # for part in sorted(parts, key=lambda x: x.part_number):
        for part in parts:
            object_parts.append({"part_number": part.part_number})

        if not object_parts:
            return False

        body = {"object_parts": object_parts}
        response = self.connection.make_request(
            "POST", self.name, key_name, data=json.dumps(body), params=params)
        if response.status == 201:
            return True
        else:
            err = get_response_error(response)
            raise err
<commit_msg>Fix typo and remove useless parameter<commit_after># =========================================================================
# Copyright 2015 Yunify, Inc.
# -------------------------------------------------------------------------
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this work except in compliance with the License.
# You may obtain a copy of the License in the LICENSE file, or at:
#
#  http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# =========================================================================

import json

from .key import Key
from .multipart import MultiPartUpload
from .exception import get_response_error

class Bucket(object):

    DefaultContentType = "application/oct-stream"

    def __init__(self, connection=None, name=None):
        self.connection = connection
        self.name = name

    def __repr__(self):
        return '<Bucket: %s>' % self.name

    def __contains__(self, key_name):
        return not (self.get_key(key_name) is None)

    def __getitem__(self, key_name):
        key = self.get_key(key_name)
        if key:
            return key
        else:
            raise KeyError

    def __len__(self):
        pass

    def get_key(self, key_name, validate=True):
        """ Retrieves an object by name.
        Returns: An instance of a Key object or None

        Keyword arguments:
        key_name - The name of the bucket
        validate - If True, the function will try to verify the object exists
            on the service-side (Default: True)
        """

        if not validate:
            return Key(self, key_name)

        response = self.connection.make_request("HEAD", self.name, key_name)

        if response.status == 200:
            return Key(self, key_name)
        elif response.status == 401:
            err = get_response_error(response)
            err.code = "invalid_access_key_id"
            err.message = "Request not authenticated, Access Key ID is either " \
                "missing or invalid."
            raise err
        elif response.status == 403:
            err = get_response_error(response)
            err.code = "permission_denied"
            err.message = "You don't have enough permission to accomplish " \
                "this request."
            raise err
        elif response.status == 404:
            err = get_response_error(response)
            err.code = "object_not_exists"
            err.message = "The object you are accessing doesn't exist."
            raise err
        else:
            err = get_response_error(response)
            raise err

    def new_key(self, key_name):
        """ Create a new object within the bucket.
        The function will not PUT an object to service-side, utils function
        send_file of the return instance is called.
        Returns: An instance of a Key object

        Keyword arguments:
        key_name - The name of the object
        """
        return Key(self, key_name)

    def delete_key(self, key_name):
        """ Deleted the particular object by object name.

        Keyword arguments:
        key_name - The name of the object
        """
        response = self.connection.make_request("DELETE", self.name, key_name)
        if response.status == 204:
            return True
        else:
            err = get_response_error(response)
            raise err

    def list(self, prefix=None, delimiter=None, marker=None, limit=None):
        """ List objects of the bucket.

        Keyword arguments:
        prefix - Limits the response to keys that begin with the specified prefix
        delimiter - A character you use to group keys
        marker - Specifies the key to start with when listing objects
        limit - The count of objects that the request returns
        """
        params = {}
        if prefix:
            params["prefix"] = prefix
        if delimiter:
            params["delimiter"] = delimiter
        if marker:
            params["marker"] = marker
        if limit:
            params["limit"] = str(limit)
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            result_set = []
            for k in resp["keys"]:
                key = Key(self, k["key"])
                key.content_type = k["mime_type"]
                result_set.append(key)
            return result_set
        else:
            err = get_response_error(response)
            raise err

    def delete(self):
        """ Delete the bucket
        """
        response = self.connection.make_request(
            "DELETE", self.name, num_retries=6)
        if response.status == 204:
            return True
        else:
            raise get_response_error(response)

    def stats(self):
        """ Retrieve the bucket meta information.
        """
        params = {"stats": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def get_acl(self):
        """ Retrieve the bucket access control list.
        """
        params = {"acl": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def set_acl(self, acls):
        """ Set the bucket access control list.

        Keyword arguments:
        acls - The list of ACL infomation
        """
        params = {"acl": None}
        data = json.dumps(acls)
        response = self.connection.make_request("PUT", self.name,
                                                params=params, data=data)
        if response.status == 200:
            return True
        else:
            err = get_response_error(response)
            raise err

    def get_cors(self):
        """ Retrieve the bucket CORS settings.
        """
        params = {"cors": None}
        response = self.connection.make_request(
            "GET", self.name, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            return resp
        else:
            err = get_response_error(response)
            raise err

    def set_cors(self, cors):
        """ Set the bucket CORS settings.

        Keyword arguments:
        cors - a dict object contains CORS rules
        """
        params = {"cors": None}
        data = json.dumps(cors)
        response = self.connection.make_request("PUT", self.name,
                                                params=params, data=data)
        if response.status == 200:
            return True
        else:
            err = get_response_error(response)
            raise err

    def delete_cors(self):
        """ Delete the bucket CORS settings.
        """
        response = self.connection.make_request("DELETE", self.name)
        if response.status == 204:
            return True
        else:
            err = get_response_error(response)
            raise err

    def initiate_multipart_upload(self, key_name, content_type=None):
        """Initiate a multipart upload.
        Returns: An instance of MultiPartUpload

        Keyword arguments:
        key_name - The object name
        content_type - The content type of the object
        """
        params = {"uploads": None}
        headers = {
            "Content-Type": content_type or self.DefaultContentType
        }
        response = self.connection.make_request(
            "POST", self.name, key_name, headers=headers, params=params)
        if response.status == 200:
            resp = json.loads(response.read())
            handler = MultiPartUpload(self, key_name, resp["upload_id"])
            return handler
        else:
            err = get_response_error(response)
            raise err

    def cancel_multipart_upload(self, key_name, upload_id):
        """Abort the multipart upload.

        Keyword arguments:
        key_name - The object name
        upload_id - ID for the initiated multipart upload
        """
        params = {
            "upload_id": upload_id,
        }
        response = self.connection.make_request(
            "DELETE", self.name, key_name, params=params)
        if response.status == 204:
            return True
        else:
            err = get_response_error(response)
            raise err

    def complete_multipart_upload(self, key_name, upload_id, parts):
        """Complete the multipart upload.

        Keyword arguments:
        key_name - The object name
        upload_id - ID for the initiated multipart upload
        parts - The list of the multiparts which need to be combined
        """
        params = {
            "upload_id": upload_id,
        }
        object_parts = []
        # for part in sorted(parts, key=lambda x: x.part_number):
        for part in parts:
            object_parts.append({"part_number": part.part_number})

        if not object_parts:
            return False

        body = {"object_parts": object_parts}
        response = self.connection.make_request(
            "POST", self.name, key_name, data=json.dumps(body), params=params)
        if response.status == 201:
            return True
        else:
            err = get_response_error(response)
            raise err
<|endoftext|>


<commit_before>import asyncio

import pytest

from aioworkers.storage import StorageError

@pytest.fixture
def aioworkers(aioworkers):
    aioworkers.plugins.append('aioworkers.net.web')
    return aioworkers

@pytest.fixture
def config_yaml(unused_port):
    return """
    http.port: {port}
    app.resources:
        /api:
            get: .data
        /api/str:
            get: .str_data
        /api/bin:
            get: .bin_data
            post: tests.test_net_web.handler_post
    data: 1
    str_data: asdf
    storage.cls: aioworkers.storage.http.Storage
    """.format(
        port=unused_port()
    )

@pytest.fixture
def config(config):
    config.update(bin_data=b'qwerty')
    return config

async def handler_post(request, context):
    body = await request.read()
    return {'body': body.decode()}

@pytest.mark.timeout(5)
async def test_web_server(context):
    await asyncio.sleep(2)
    url = context.http.url
    assert 1 == await context.storage.get(url / 'api')
    assert 'asdf' == await context.storage.get(url / 'api/str')
    with pytest.raises(StorageError):
        await context.storage.set(url / 'api/str', b'123')  # 405
    assert b'qwerty' == await context.storage.get(url / 'api/bin')
    d = await context.storage.set(url / 'api/bin', b'123')
    assert d == {'body': '123'}
    assert not await context.storage.set(url / 'api/not/found', b'123')  # 404
<commit_msg>drop sleep from test_web_server<commit_after>import asyncio

import pytest

from aioworkers.storage import StorageError

@pytest.fixture
def aioworkers(aioworkers):
    aioworkers.plugins.append('aioworkers.net.web')
    return aioworkers

@pytest.fixture
def config_yaml(unused_port):
    return """
    http.port: {port}
    app.resources:
        /api:
            get: .data
        /api/str:
            get: .str_data
        /api/bin:
            get: .bin_data
            post: tests.test_net_web.handler_post
    data: 1
    str_data: asdf
    storage.cls: aioworkers.storage.http.Storage
    """.format(
        port=unused_port()
    )

@pytest.fixture
def config(config):
    config.update(bin_data=b'qwerty')
    return config

async def handler_post(request, context):
    body = await request.read()
    return {'body': body.decode()}

@pytest.mark.timeout(5)
async def test_web_server(context):
    url = context.http.url
    assert 1 == await context.storage.get(url / 'api')
    assert 'asdf' == await context.storage.get(url / 'api/str')
    with pytest.raises(StorageError):
        await context.storage.set(url / 'api/str', b'123')  # 405
    assert b'qwerty' == await context.storage.get(url / 'api/bin')
    d = await context.storage.set(url / 'api/bin', b'123')
    assert d == {'body': '123'}
    assert not await context.storage.set(url / 'api/not/found', b'123')  # 404
<|endoftext|>


<commit_before>"""
OHSU - This module wraps the proprietary OHSU AIRC ``fastfit``
software. ``fastfit`` optimizes the input pharmacokinetic model.

:Note: this interface is adapted from the OHSU AIRC cluster file
    ``/usr/global/scripts/fastfit_iface.py``.
"""

# TODO - move this to the ohsu-qin.pipeline.interfaces module

# Note - as of Jun 2017, the old fastfit is broken and the new,
# improved fastfit is only available in a virtual environment.
# Therefore, it is necessary to override the global fastfit
# in the PATH with the following local fastfit script:
#
# #! env bash
#
# # Reset the path. If the system login path was captured in the
# # SYS_PATH envvar (preferred), then use that as the basis.
# # Otherwise, take our chances with the current PATH.
# if [ -n "$SYS_PATH" ]; then
#     PATH="/usr/global/venvs/fastfit/bin:$SYS_PATH"
# else
#     PATH="/usr/global/venvs/fastfit/bin:$PATH"
# fi
#
# # Borrowed from venv activate:
# # This should detect bash and zsh, which have a hash command
# # that must be called to get it to forget past commands.
# # Without forgetting past commands the $PATH changes we made
# # may not be respected.
# if [ -n "$BASH" -o -n "$ZSH_VERSION" ]; then
#     hash -r
# fi
#
# echo "Submitting `which mpiexec` `which fastfit` $*..."
# mpiexec fastfit $*
#
# TODO - Once /usr/global/bin/fastfit is upgraded, derive Fastfit
# from MpiCommand.
#

# Absolute import (the default in a future Python release) resolves
# the fastfit import as the installed fastfit Python module rather
# than this interface of the same name.
from __future__ import absolute_import
import os
from os import path
from glob import glob
from twisted.python.procutils import which
from nipype.interfaces.base import (
    traits, DynamicTraitedSpec, CommandLine, CommandLineInputSpec,
    isdefined
)
from nipype.interfaces.traits_extension import Undefined
from .interface_error import InterfaceError
from ..helpers.logging import logger

class FastfitInputSpec(CommandLineInputSpec):
    model_name = traits.String(desc='The name of the model to optimize',
                               mandatory=True, position=-2, argstr='%s')

    in_file = traits.File(
        desc='The source 4D NIfTI file containing the target data',
        mandatory=True, position=-1, argstr='%s'
    )

    mask = traits.File(desc='Mask file', argstr='-m %s')

    weights = traits.File(desc='Weights file', argstr='-w %s')

    optimization_params = traits.List(
        desc='The required optimization parameters for the model',
        mandatory=True
    )

    other_params = traits.Dict(desc='Other parameters for the model')

    other_params_csv = traits.File(desc='Other parameters CSV',
                                   argstr='--param-csv %s')

    fix_params = traits.Dict(desc="Optimization parameters to fix, and "
                             "the values to fix them to", argstr='%s')

    optional_outs = traits.List(desc='Optional outputs to produce',
                                argstr='%s')

class Fastfit(CommandLine):
    """
    Interface to the ``fastfit`` software package.
    """

    _cmd = 'fastfit'

    input_spec = FastfitInputSpec

    output_spec = DynamicTraitedSpec

    def __init__(self, **inputs):
        super(Fastfit, self).__init__(**inputs)

    # The cmdline override below is commented out, since it only
    # applies to the broken old fastfit. However, it may apply to
    # the new fastfit when it replaces the global fastfit.
    # # Note: nipype MpiCommandLine calls mpiexec, which resolves to the
    # # local build mpiexec. This results in MPICOMM errors. The
    # # work-around is to replace mpiexec with the absolute path of
    # # the official mpiexec.
    # @property
    # def cmdline(self):
    #     result = super(Fastfit, self).cmdline
    #     # Delay until this last possible moment the fastfit existence
    #     # check. This delay allows Fastfit interface creation without
    #     # the fastfit executable, e.g. in a dry run.
    #     matches = which('fastfit')
    #     if matches:
    #         logger(__name__).debug("The fastfit executable is %s" %
    #                                matches[0])
    #     else:
    #         raise InterfaceError("No fastfit executable was found in"
    #                              " the path %s" % os.environ('PATH'))
    #
    #     # MpiCommandLine wraps the command in a call to mpiexec.
    #     # In the OHSU AIRC context, the only mpiexec which can
    #     # run fastfit is in /usr/global/bin. Anaconda installs its
    #     # own mpiexec, which cannot be used because of a loadlib
    #     # incompatibility.
    #     return result.replace('mpiexec', '/usr/global/bin/mpiexec')

    def _format_arg(self, name, spec, value):
        if name == 'optional_outs':
            return ' '.join("-o '%s'" % opt_out for opt_out in value)
        elif name == 'other_params':
            return ' '.join(["--set-param %s:%s" % item
                             for item in value.iteritems()])
        elif name == 'fix_params':
            return ' '.join(["--fix-param %s:%s" % item
                             for item in value.iteritems()])
        else:
            return spec.argstr % value

    def _outputs(self):
        # Set up dynamic outputs for resulting parameter maps.
        outputs = super(Fastfit, self)._outputs()
        undefined_traits = {}

        # Get a list of fixed parameters.
        fixed_params = []
        if isdefined(self.inputs.fix_params):
            for key in self.inputs.fix_params:
                fixed_params.append(key)

        # Add optimization parameter outputs.
        self._opt_params = self.inputs.optimization_params
        for param_name in self._opt_params:
            if not param_name in fixed_params:
                outputs.add_trait(param_name, traits.File(exists=True))
                undefined_traits[param_name] = Undefined

        # Add optional outputs.
        if isdefined(self.inputs.optional_outs):
            for opt_out in self.inputs.optional_outs:
                outputs.add_trait(opt_out, traits.File(exists=True))
                undefined_traits[opt_out] = Undefined

        # Mystery code retained from fastfit_iface.py.
        outputs.trait_set(trait_change_notify=False, **undefined_traits)
        for dynamic_out in undefined_traits.keys():
            _ = getattr(outputs, dynamic_out)

        return outputs

    def _list_outputs(self):
        outputs = self._outputs().get()

        cwd = os.getcwd()
        for param_name in outputs.keys():
            outputs[param_name] = path.join(cwd, '%s_map.nii.gz' % param_name)

        return outputs
<commit_msg>Clarify comment.<commit_after>"""
OHSU - This module wraps the proprietary OHSU AIRC ``fastfit``
software. ``fastfit`` optimizes the input pharmacokinetic model.

:Note: this interface is adapted from the OHSU AIRC cluster file
    ``/usr/global/scripts/fastfit_iface.py``.
"""

# TODO - move this to the ohsu-qin.pipeline.interfaces module

# Note - as of Jun 2017, the old fastfit is broken and the new,
# improved fastfit is only available in a virtual environment.
# Therefore, it is necessary to override the global fastfit
# in the PATH with the following local fastfit script:
#
# #! env bash
#
# # Reset the path. If the system login path was captured in the
# # SYS_PATH envvar (preferred), then use that as the basis.
# # Otherwise, take our chances with the current PATH.
# if [ -n "$SYS_PATH" ]; then
#     PATH="/usr/global/venvs/fastfit/bin:$SYS_PATH"
# else
#     PATH="/usr/global/venvs/fastfit/bin:$PATH"
# fi
#
# # Borrowed from venv activate:
# # This should detect bash and zsh, which have a hash command
# # that must be called to get it to forget past commands.
# # Without forgetting past commands the $PATH changes we made
# # may not be respected.
# if [ -n "$BASH" -o -n "$ZSH_VERSION" ]; then
#     hash -r
# fi
#
# echo "Submitting `which mpiexec` `which fastfit` $*..."
# mpiexec fastfit $*
#
# TODO - Once /usr/global/bin/fastfit is upgraded, derive Fastfit
# from MpiCommand.
#

# Absolute import (the default in a future Python release) resolves
# the fastfit import as the installed fastfit Python module rather
# than this interface of the same name.
from __future__ import absolute_import
import os
from os import path
from glob import glob
from twisted.python.procutils import which
from nipype.interfaces.base import (
    traits, DynamicTraitedSpec, CommandLine, CommandLineInputSpec,
    isdefined
)
from nipype.interfaces.traits_extension import Undefined
from .interface_error import InterfaceError
from ..helpers.logging import logger

class FastfitInputSpec(CommandLineInputSpec):
    model_name = traits.String(desc='The name of the model to optimize',
                               mandatory=True, position=-2, argstr='%s')

    in_file = traits.File(
        desc='The source 4D NIfTI file containing the target data',
        mandatory=True, position=-1, argstr='%s'
    )

    mask = traits.File(desc='Mask file', argstr='-m %s')

    weights = traits.File(desc='Weights file', argstr='-w %s')

    optimization_params = traits.List(
        desc='The required optimization parameters for the model',
        mandatory=True
    )

    other_params = traits.Dict(desc='Other parameters for the model')

    other_params_csv = traits.File(desc='Other parameters CSV',
                                   argstr='--param-csv %s')

    fix_params = traits.Dict(desc="Optimization parameters to fix, and "
                             "the values to fix them to", argstr='%s')

    optional_outs = traits.List(desc='Optional outputs to produce',
                                argstr='%s')

class Fastfit(CommandLine):
    """
    Interface to the ``fastfit`` software package.
    """

    _cmd = 'fastfit'

    input_spec = FastfitInputSpec

    output_spec = DynamicTraitedSpec

    def __init__(self, **inputs):
        super(Fastfit, self).__init__(**inputs)

    # The cmdline override below is commented out, since it only
    # applies to the broken old fastfit. However, it may apply to
    # the new fastfit when it replaces the global fastfit.
    #
    # # Note: nipype MpiCommandLine calls mpiexec, which resolves
    # # to the Anaconda virtual environment mpiexec. This results
    # # in MPICOMM errors. The work-around is to replace mpiexec
    # # in the command with the absolute path of the global mpiexec.
    # @property
    # def cmdline(self):
    #     result = super(Fastfit, self).cmdline
    #     # Delay until this last possible moment the fastfit existence
    #     # check. This delay allows Fastfit interface creation without
    #     # the fastfit executable, e.g. in a dry run.
    #     matches = which('fastfit')
    #     if matches:
    #         logger(__name__).debug("The fastfit executable is %s" %
    #                                matches[0])
    #     else:
    #         raise InterfaceError("No fastfit executable was found in"
    #                              " the path %s" % os.environ('PATH'))
    #
    #     # MpiCommandLine wraps the command in a call to mpiexec.
    #     # In the OHSU AIRC context, the only mpiexec which can
    #     # run fastfit is in /usr/global/bin. Anaconda installs its
    #     # own mpiexec, which cannot be used because of a loadlib
    #     # incompatibility.
    #     return result.replace('mpiexec', '/usr/global/bin/mpiexec')

    def _format_arg(self, name, spec, value):
        if name == 'optional_outs':
            return ' '.join("-o '%s'" % opt_out for opt_out in value)
        elif name == 'other_params':
            return ' '.join(["--set-param %s:%s" % item
                             for item in value.iteritems()])
        elif name == 'fix_params':
            return ' '.join(["--fix-param %s:%s" % item
                             for item in value.iteritems()])
        else:
            return spec.argstr % value

    def _outputs(self):
        # Set up dynamic outputs for resulting parameter maps.
        outputs = super(Fastfit, self)._outputs()
        undefined_traits = {}

        # Get a list of fixed parameters.
        fixed_params = []
        if isdefined(self.inputs.fix_params):
            for key in self.inputs.fix_params:
                fixed_params.append(key)

        # Add optimization parameter outputs.
        self._opt_params = self.inputs.optimization_params
        for param_name in self._opt_params:
            if not param_name in fixed_params:
                outputs.add_trait(param_name, traits.File(exists=True))
                undefined_traits[param_name] = Undefined

        # Add optional outputs.
        if isdefined(self.inputs.optional_outs):
            for opt_out in self.inputs.optional_outs:
                outputs.add_trait(opt_out, traits.File(exists=True))
                undefined_traits[opt_out] = Undefined

        # Mystery code retained from fastfit_iface.py.
        outputs.trait_set(trait_change_notify=False, **undefined_traits)
        for dynamic_out in undefined_traits.keys():
            _ = getattr(outputs, dynamic_out)

        return outputs

    def _list_outputs(self):
        outputs = self._outputs().get()

        cwd = os.getcwd()
        for param_name in outputs.keys():
            outputs[param_name] = path.join(cwd, '%s_map.nii.gz' % param_name)

        return outputs
<|endoftext|>


<commit_before>import inspect

from approvaltests import (
    get_default_reporter,
    approvals,
    verify,
    ReporterForTesting,
    combination_approvals,
)
from approvaltests.core.options import Options
from approvaltests.mrjob import mrjob_approvals
from approvaltests.utilities import command_line_approvals
from approvaltests.utilities.logger import simple_logger_approvals

def test_every_function_in_approvals_with_verify_has_an_options():
    assert_verify_methods_have_options(approvals)
    assert_verify_methods_have_options(combination_approvals)
    assert_verify_methods_have_options(simple_logger_approvals)
    assert_verify_methods_have_options(command_line_approvals)
    assert_verify_methods_have_options(mrjob_approvals)

def assert_verify_methods_have_options(module):
    for function_name, obj in module.__dict__.items():
        if "verify" not in function_name:
            continue

        if not callable(obj):
            continue

        argspec = inspect.getfullargspec(obj)
        print(f"{function_name}: {argspec}")
        has_options = "options" in argspec.kwonlyargs # Diana want could have said got_options ? like got_milk ?
        assert (has_options), f"In {function_name}"

def test_empty_options_has_default_reporter():
    ##approvals.set_default_reporter(None)
    options = Options()
    assert options.reporter == get_default_reporter()

def test_with_reporter():
    testr = ReporterForTesting()
    options = Options().with_reporter(testr)
    try:
        verify("Data2", options=options)
    except:
        pass

    assert testr.called

def test_setting_reporter():
    testr = ReporterForTesting()
    options = Options().with_reporter(testr)
    assert options.reporter == testr

def test_file_extensions():
    content = "# This is a markdown header\n"
    # begin-snippet: options_with_file_extension
    verify(content, options=Options().for_file.with_extension(".md"))
    # end-snippet
    verify(content, options=Options().for_file.with_extension("md"))
<commit_msg>r clutter<commit_after>import inspect

from approvaltests import (
    get_default_reporter,
    approvals,
    verify,
    ReporterForTesting,
    combination_approvals,
)
from approvaltests.core.options import Options
from approvaltests.mrjob import mrjob_approvals
from approvaltests.utilities import command_line_approvals
from approvaltests.utilities.logger import simple_logger_approvals

def test_every_function_in_approvals_with_verify_has_an_options():
    assert_verify_methods_have_options(approvals)
    assert_verify_methods_have_options(combination_approvals)
    assert_verify_methods_have_options(simple_logger_approvals)
    assert_verify_methods_have_options(command_line_approvals)
    assert_verify_methods_have_options(mrjob_approvals)

def assert_verify_methods_have_options(module):
    for function_name, obj in module.__dict__.items():
        if "verify" not in function_name:
            continue

        if not callable(obj):
            continue

        argspec = inspect.getfullargspec(obj)
        print(f"{function_name}: {argspec}")
        has_options = "options" in argspec.kwonlyargs
        assert (has_options), f"In {function_name}"

def test_empty_options_has_default_reporter():
    ##approvals.set_default_reporter(None)
    options = Options()
    assert options.reporter == get_default_reporter()

def test_with_reporter():
    testr = ReporterForTesting()
    options = Options().with_reporter(testr)
    try:
        verify("Data2", options=options)
    except:
        pass

    assert testr.called

def test_setting_reporter():
    testr = ReporterForTesting()
    options = Options().with_reporter(testr)
    assert options.reporter == testr

def test_file_extensions():
    content = "# This is a markdown header\n"
    # begin-snippet: options_with_file_extension
    verify(content, options=Options().for_file.with_extension(".md"))
    # end-snippet
    verify(content, options=Options().for_file.with_extension("md"))
<|endoftext|>


<commit_before>"""
This module wraps the proprietary OHSU AIRC ``fastfit`` software.
``fastfit`` optimizes the input pharmacokinetic model.

:Note: this interface is copied from the AIRC cluster
    ``/usr/global/scripts/fastfit_iface.py`` file. It is included
    in :mod:`qipipe.interfaces` in order to gather all custom OHSU
    QIN Python source code in the ``qipipe`` module. If it were not
    included, then qipipe would not compile in a non-AIRC cluster
    environment, even if this ``FastFit`` interface is unused.
"""
from __future__ import absolute_import

import os
from os import path
from glob import glob
import traits.api as traits
from nipype.interfaces.base import (DynamicTraitedSpec,
                                    CommandLine,
                                    CommandLineInputSpec,
                                    isdefined)
from nipype.interfaces.traits_extension import Undefined
from fastfit.fastfit_cli import get_available_models

class MpiCommandLineInputSpec(CommandLineInputSpec):
    use_mpi = traits.Bool(False,
                          desc='Whether or not to run the command with mpiexec',
                          usedefault=True)
    n_procs = traits.Int(desc='Num processors to specify to mpiexec')

class MpiCommandLine(CommandLine):
    @property
    def cmdline(self):
        """Adds 'mpiexec' to begining of command"""
        result = []
        if self.inputs.use_mpi:
            result.append('mpiexec')
            if self.inputs.n_procs:
                result.append('-n %d' % self.inputs.n_procs)
        result.append(super(MpiCommandLine, self).cmdline)
        return ' '.join(result)

class FastfitInputSpec(MpiCommandLineInputSpec):
    model_name = traits.String(desc='The name of the model to optimize',
                               mandatory=True,
                               position=-2,
                               argstr='%s'
                              )
    target_data = traits.File(desc='Target data',
                              mandatory=True,
                              position=-1,
                              argstr='%s'
                             )
    mask = traits.File(desc='Mask file', argstr='-m %s')
    weights = traits.File(desc='Weights file', argstr='-w %s')
    params = traits.Dict(desc='Parameters for the model')
    params_csv = traits.File(desc='Parameters CSV',
                             argstr='--param-csv %s')
    fix_params = traits.Dict(desc="Optimization parameters to fix, and "
                             "the values to fix them to", argstr='%s')
    optional_outs = traits.List(desc='Optional outputs to produce',
                                argstr='%s')

class Fastfit(MpiCommandLine):
    '''Interface to the "fastfit" software package.

    Parameters
    ----------
    min_outs : list
        The minimum outputs expected. Required if the 'model_name'
        input is being set dynamically.
    '''

    _cmd = 'fastfit'
    input_spec = FastfitInputSpec
    output_spec = DynamicTraitedSpec

    def __init__(self, min_outs=None, **inputs):
        super(Fastfit, self).__init__(**inputs)
        self._min_outs = min_outs

    def _format_arg(self, name, spec, value):
        if name == 'optional_outs':
            return ' '.join("-o '%s'" % opt_out for opt_out in value)
        elif name == 'params':
            return ' '.join(["-s %s:%s" % item
                             for item in value.iteritems()])
        elif name == 'fix_params':
            return ' '.join(["--fix-param %s:%s" % item
                             for item in value.iteritems()])
        else:
            return spec.argstr % value

    def _outputs(self):
        #Set up dynamic outputs for resulting parameter maps
        outputs = super(Fastfit, self)._outputs()
        undefined_traits = {}

        #Get a list of fixed params
        fixed_params = []
        if isdefined(self.inputs.fix_params):
            for key in self.inputs.fix_params:
                fixed_params.append(key)

        #If the model name is static, we can find the outputs
        if isdefined(self.inputs.model_name):
            model_mods, builtin_models, user_models = get_available_models()
            model = model_mods[self.inputs.model_name]
            self._opt_params = model.optimization_params
            if (not self._min_outs is None and
                any(not out in self._opt_params
                    for out in self._min_outs)
               ):
                raise ValueError("The model %s does not provide the "
                                 "minimum outputs" % model_name)
            for param_name in self._opt_params:
                if not param_name in fixed_params:
                    outputs.add_trait(param_name, traits.File(exists=True))
                    undefined_traits[param_name] = Undefined

        #Otherwise we need min_outs
        elif not self._min_outs is None:
            for param_name in self._min_outs:
                outputs.add_trait(param_name, traits.File(exists=True))
                undefined_traits[param_name] = Undefined
        else:
            raise ValueError("Either the 'model_name' input must "
                             "be static or the 'min_outs' argument "
                             "to the constructor must be given.")

        #Set up dynamic outputs for any requested optional outputs
        if isdefined(self.inputs.optional_outs):
            for opt_out in self.inputs.optional_outs:
                outputs.add_trait(opt_out, traits.File(exists=True))
                undefined_traits[opt_out] = Undefined

        outputs.trait_set(trait_change_notify=False, **undefined_traits)
        for dynamic_out in undefined_traits.keys():
            _ = getattr(outputs, dynamic_out) #Not sure why, but this is needed
        return outputs

    def _list_outputs(self):
        outputs = self._outputs().get()

        cwd = os.getcwd()
        for param_name in outputs.keys():
            outputs[param_name] = path.join(cwd,
                                            '%s_map.nii.gz' % param_name)

        return outputs
<commit_msg>Add comment.<commit_after>"""
This module wraps the proprietary OHSU AIRC ``fastfit`` software.
``fastfit`` optimizes the input pharmacokinetic model.

:Note: this interface is copied from the AIRC cluster
    ``/usr/global/scripts/fastfit_iface.py`` file. It is included
    in :mod:`qipipe.interfaces` in order to gather all custom OHSU
    QIN Python source code in the ``qipipe`` module. If it were not
    included, then qipipe would not compile in a non-AIRC cluster
    environment, even if this ``FastFit`` interface is unused.
"""
# Absolute import (the default in a future Python release) resolves
# the fastfit import as the installed fastfit Python module
# rather than this interface of the same name.
from __future__ import absolute_import

import os
from os import path
from glob import glob
import traits.api as traits
from nipype.interfaces.base import (DynamicTraitedSpec,
                                    CommandLine,
                                    CommandLineInputSpec,
                                    isdefined)
from nipype.interfaces.traits_extension import Undefined
from fastfit.fastfit_cli import get_available_models

class MpiCommandLineInputSpec(CommandLineInputSpec):
    use_mpi = traits.Bool(False,
                          desc='Whether or not to run the command with mpiexec',
                          usedefault=True)
    n_procs = traits.Int(desc='Num processors to specify to mpiexec')

class MpiCommandLine(CommandLine):
    @property
    def cmdline(self):
        """Adds 'mpiexec' to begining of command"""
        result = []
        if self.inputs.use_mpi:
            result.append('mpiexec')
            if self.inputs.n_procs:
                result.append('-n %d' % self.inputs.n_procs)
        result.append(super(MpiCommandLine, self).cmdline)
        return ' '.join(result)

class FastfitInputSpec(MpiCommandLineInputSpec):
    model_name = traits.String(desc='The name of the model to optimize',
                               mandatory=True,
                               position=-2,
                               argstr='%s'
                              )
    target_data = traits.File(desc='Target data',
                              mandatory=True,
                              position=-1,
                              argstr='%s'
                             )
    mask = traits.File(desc='Mask file', argstr='-m %s')
    weights = traits.File(desc='Weights file', argstr='-w %s')
    params = traits.Dict(desc='Parameters for the model')
    params_csv = traits.File(desc='Parameters CSV',
                             argstr='--param-csv %s')
    fix_params = traits.Dict(desc="Optimization parameters to fix, and "
                             "the values to fix them to", argstr='%s')
    optional_outs = traits.List(desc='Optional outputs to produce',
                                argstr='%s')

class Fastfit(MpiCommandLine):
    '''Interface to the "fastfit" software package.

    Parameters
    ----------
    min_outs : list
        The minimum outputs expected. Required if the 'model_name'
        input is being set dynamically.
    '''

    _cmd = 'fastfit'
    input_spec = FastfitInputSpec
    output_spec = DynamicTraitedSpec

    def __init__(self, min_outs=None, **inputs):
        super(Fastfit, self).__init__(**inputs)
        self._min_outs = min_outs

    def _format_arg(self, name, spec, value):
        if name == 'optional_outs':
            return ' '.join("-o '%s'" % opt_out for opt_out in value)
        elif name == 'params':
            return ' '.join(["-s %s:%s" % item
                             for item in value.iteritems()])
        elif name == 'fix_params':
            return ' '.join(["--fix-param %s:%s" % item
                             for item in value.iteritems()])
        else:
            return spec.argstr % value

    def _outputs(self):
        #Set up dynamic outputs for resulting parameter maps
        outputs = super(Fastfit, self)._outputs()
        undefined_traits = {}

        #Get a list of fixed params
        fixed_params = []
        if isdefined(self.inputs.fix_params):
            for key in self.inputs.fix_params:
                fixed_params.append(key)

        #If the model name is static, we can find the outputs
        if isdefined(self.inputs.model_name):
            model_mods, builtin_models, user_models = get_available_models()
            model = model_mods[self.inputs.model_name]
            self._opt_params = model.optimization_params
            if (not self._min_outs is None and
                any(not out in self._opt_params
                    for out in self._min_outs)
               ):
                raise ValueError("The model %s does not provide the "
                                 "minimum outputs" % model_name)
            for param_name in self._opt_params:
                if not param_name in fixed_params:
                    outputs.add_trait(param_name, traits.File(exists=True))
                    undefined_traits[param_name] = Undefined

        #Otherwise we need min_outs
        elif not self._min_outs is None:
            for param_name in self._min_outs:
                outputs.add_trait(param_name, traits.File(exists=True))
                undefined_traits[param_name] = Undefined
        else:
            raise ValueError("Either the 'model_name' input must "
                             "be static or the 'min_outs' argument "
                             "to the constructor must be given.")

        #Set up dynamic outputs for any requested optional outputs
        if isdefined(self.inputs.optional_outs):
            for opt_out in self.inputs.optional_outs:
                outputs.add_trait(opt_out, traits.File(exists=True))
                undefined_traits[opt_out] = Undefined

        outputs.trait_set(trait_change_notify=False, **undefined_traits)
        for dynamic_out in undefined_traits.keys():
            _ = getattr(outputs, dynamic_out) #Not sure why, but this is needed
        return outputs

    def _list_outputs(self):
        outputs = self._outputs().get()

        cwd = os.getcwd()
        for param_name in outputs.keys():
            outputs[param_name] = path.join(cwd,
                                            '%s_map.nii.gz' % param_name)

        return outputs
<|endoftext|>


<commit_before>import os

from pch2csd.parsing.parsing import parse_pch2
from pch2csd.parsing.structs import Module, Location, Cable, CableType, CableColor

def get_test_resource(path: str) -> str:
    return os.path.join(os.path.dirname(__file__), 'resources', path)

class TestParsing(TestCase):
    def test_gleb2_pch(self):
        gleb_2_pch = get_test_resource('Gleb2.pch2')
        expected_modules = [Module(Location.VOICE_AREA, 4, 1),
                            Module(Location.VOICE_AREA, 96, 2)]
        expected_cables = [
            Cable(Location.VOICE_AREA, CableType.IN_TO_IN, CableColor.WHITE, 1, 0, 1, 1)]
        parsed = parse_pch2(gleb_2_pch)
        self.assertSequenceEqual(parsed.modules, expected_modules)
        self.assertSequenceEqual(parsed.cables, expected_cables)

<commit_msg>[fix] imports in test_parsing<commit_after>import os
from unittest import TestCase

from pch2csd.parsing.parsing import parse_pch2
from pch2csd.parsing.structs import Module, Location, Cable, CableType, CableColor

def get_test_resource(path: str) -> str:
    return os.path.join(os.path.dirname(__file__), 'resources', path)

class TestParsing(TestCase):
    def test_gleb2_pch(self):
        gleb_2_pch = get_test_resource('Gleb2.pch2')
        expected_modules = [Module(Location.VOICE_AREA, 4, 1),
                            Module(Location.VOICE_AREA, 96, 2)]
        expected_cables = [
            Cable(Location.VOICE_AREA, CableType.IN_TO_IN, CableColor.WHITE, 1, 0, 1, 1)]
        parsed = parse_pch2(gleb_2_pch)
        self.assertSequenceEqual(parsed.modules, expected_modules)
        self.assertSequenceEqual(parsed.cables, expected_cables)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_revolver
----------------------------------

Tests for `pathmap` module.
"""

import pytest

from pathmap import (
    clean_path,
    _slash_pattern,
    _extract_match,
    _resolve_path,
    _resolve_path_if_long,
    _check_ancestors,
    resolve_paths,
    resolve_by_method
)

from lcs import longest_common_substring

# ========== Mock data ===========
before = [
    'before/path.py',
    'before/path.py',
    'not/found.py',
    '/Users/user/owner/repo/dist/components/login.js',
    'site-packages/package/__init__.py',
    'path.py',
    'a/b/../Path With\\ Space'
]

after = [
    'after/path.py',
    'after/path.py',
    None,
    'src/components/login.js',
    'package/__init__.py',
    'path.py',
    'a/Path With Space'
]

toc = ','.join(map(lambda x: "" if x is None else x, after)) + ','

# ========= END Mock data ==========
def test_clean_path():
    path = '**/some/directory'
    assert clean_path(path) == 'some/directory'
    path = 'some/path\r/with/tabs\r'
    assert clean_path(path) == 'some/path/with/tabs'
    path = 'some\ very_long/directory\ name'
    assert clean_path(path) == 'some very_long/directory name'
    path = 'ms\\style\\directory'
    assert clean_path(path) == 'ms/style/directory'

def test_slash_pattern():
    has_slash = 'slash/'
    assert _slash_pattern(has_slash) == 'slash/'

def test_extract_match():
    toc = ',src/components/login.js,'
    index = toc.find('components')
    extracted = _extract_match(toc, index)
    assert extracted == 'src/components/login.js'

def test_longest_common_substring():
    paths = ','.join(before)
    result = longest_common_substring('some/folder/../repo/dist/components/login.js', paths)
    assert result == '/repo/dist/components/login.js'

def test_resolve_path_if_long():
    path = '/Users/user/owner/repo/dist/components/login.js'
    (new_path, pattern) = _resolve_path_if_long(toc, path)
    assert new_path == 'src/components/login.js'
    assert pattern == ('/Users/user/owner/repo/dist/', 'src/')

def test_resolve_path_if_long_empty():
    (new_path, pattern) = _resolve_path_if_long('', '')
    assert (new_path, pattern) == (None, None)
    (new_path, pattern) = _resolve_path_if_long('abc/xyz', 'abc')
    assert (new_path, pattern) == (None, None)

def test_resolve_path():
    # short to long
    path = 'components/login.js'
    (new_path, pattern) = _resolve_path(toc, path, [])
    assert new_path == 'src/components/login.js'
    assert pattern == ('', 'src/')

def test_resolve_paths():
    resolved_paths = resolve_paths(toc, before)
    first = set([r for r in resolved_paths])
    second = set(after)
    assert first == second

def test_resolve_by_method():
    resolver = resolve_by_method(toc)
    assert callable(resolver)
    first = set(map(resolver, before))
    second = set(after)
    assert first == second

def test_check_ancestors():
    ancestors = 1
    path = 'one/two/three'
    match = 'four/two/three'
    assert _check_ancestors(path, match, ancestors) is True
    match = 'four/five/three'
    assert _check_ancestors(path, match, ancestors) is False

def test_resolve_paths_with_ancestors():
    toc = ',x/y/z,'

    # default, no ancestors ============================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = ['x/y/z', 'x/y/z', 'x/y/z', 'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths))
    assert set(resolved) == set(expected)

    # one ancestors ====================================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = [None, None, 'x/y/z', 'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths, 1))
    assert set(resolved) == set(expected)

    # two ancestors ====================================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = [None, None, None,  'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths, 2))
    assert set(resolved) == set(expected)

def test_case_sensitive_ancestors():
    toc = ',src/HeapDump/GCHeapDump.cs,'
    path = 'C:/projects/perfview/src/heapDump/GCHeapDump.cs'
    (path, pattern) = _resolve_path_if_long(toc, path, 1)
    assert path == 'src/HeapDump/GCHeapDump.cs'

def test_path_should_not_resolve():
    resolvers = []
    toc = ',four/six/three.py,'
    path = 'four/six/seven.py'
    (path, pattern) = _resolve_path(toc, path, resolvers)
    assert path is None
    assert pattern is None
<commit_msg>replace issue<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
test_revolver
----------------------------------

Tests for `pathmap` module.
"""

import pytest

from pathmap import (
    clean_path,
    _slash_pattern,
    _extract_match,
    _resolve_path,
    _resolve_path_if_long,
    _check_ancestors,
    resolve_paths,
    resolve_by_method
)

from lcs import longest_common_substring

# ========== Mock data ===========
before = [
    'before/path.py',
    'before/path.py',
    'not/found.py',
    '/Users/user/owner/repo/dist/components/login.js',
    'site-packages/package/__init__.py',
    'path.py',
    'a/b/../Path With\\ Space'
]

after = [
    'after/path.py',
    'after/path.py',
    None,
    'src/components/login.js',
    'package/__init__.py',
    'path.py',
    'a/Path With Space'
]

toc = ','.join(map(lambda x: "" if x is None else x, after)) + ','

# ========= END Mock data ==========
def test_clean_path():
    path = '**/some/directory'
    assert clean_path(path) == 'some/directory'
    path = 'some/path\r/with/tabs\r'
    assert clean_path(path) == 'some/path/with/tabs'
    path = 'some\ very_long/directory\ name'
    assert clean_path(path) == 'some very_long/directory name'
    path = 'ms\\style\\directory'
    assert clean_path(path) == 'ms/style/directory'

def test_slash_pattern():
    has_slash = 'slash/'
    assert _slash_pattern(has_slash) == 'slash/'

def test_extract_match():
    toc = ',src/components/login.js,'
    index = toc.find('components')
    extracted = _extract_match(toc, index)
    assert extracted == 'src/components/login.js'

def test_longest_common_substring():
    paths = ','.join(before)
    result = longest_common_substring('some/folder/../repo/dist/components/login.js', paths)
    assert result == '/repo/dist/components/login.js'

def test_resolve_path_if_long():
    path = '/Users/user/owner/repo/dist/components/login.js'
    (new_path, pattern) = _resolve_path_if_long(toc, path)
    assert new_path == 'src/components/login.js'
    assert pattern == ('/Users/user/owner/repo/dist/', 'src/')

def test_resolve_path_if_long_empty():
    (new_path, pattern) = _resolve_path_if_long('', '')
    assert (new_path, pattern) == (None, None)
    (new_path, pattern) = _resolve_path_if_long('abc/xyz', 'abc')
    assert (new_path, pattern) == (None, None)

def test_resolve_path():
    # short to long
    path = 'components/login.js'
    (new_path, pattern) = _resolve_path(toc, path, [])
    assert new_path == 'src/components/login.js'
    assert pattern == ('', 'src/')

def test_resolve_paths():
    resolved_paths = resolve_paths(toc, before)
    first = set([r for r in resolved_paths])
    second = set(after)
    assert first == second

def test_resolve_by_method():
    resolver = resolve_by_method(toc)
    assert callable(resolver)
    first = set(map(resolver, before))
    second = set(after)
    assert first == second

def test_check_ancestors():
    ancestors = 1
    path = 'one/two/three'
    match = 'four/two/three'
    assert _check_ancestors(path, match, ancestors) is True
    match = 'four/five/three'
    assert _check_ancestors(path, match, ancestors) is False

def test_resolve_paths_with_ancestors():
    toc = ',x/y/z,'

    # default, no ancestors ============================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = ['x/y/z', 'x/y/z', 'x/y/z', 'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths))
    assert set(resolved) == set(expected)

    # one ancestors ====================================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = [None, None, 'x/y/z', 'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths, 1))
    assert set(resolved) == set(expected)

    # two ancestors ====================================
    paths = ['z', 'R/z', 'R/y/z', 'x/y/z', 'w/x/y/z']
    expected = [None, None, None,  'x/y/z', 'x/y/z']
    resolved = list(resolve_paths(toc, paths, 2))
    assert set(resolved) == set(expected)

def test_case_sensitive_ancestors():
    toc = ',src/HeapDump/GCHeapDump.cs,'
    path = 'C:/projects/perfview/src/heapDump/GCHeapDump.cs'
    (path, pattern) = _resolve_path_if_long(toc, path, 1)
    assert path == 'src/HeapDump/GCHeapDump.cs'

def test_path_should_not_resolve():
    resolvers = []
    toc = ',four/six/three.py,'
    path = 'four/six/seven.py'
    (path, pattern) = _resolve_path(toc, path, resolvers)
    assert path is None
    assert pattern is None

def test_path_should_not_resolve_case_insensative():
    resolvers = []
    toc = ',a/b/C,'
    path = 'a/B/c'
    (path, pattern) = _resolve_path(toc, path, resolvers)
    assert path == 'a/b/C'
    assert pattern == ('a/B/', 'a/b/')
<|endoftext|>