<commit_before>import unittest
import os
import io
import re
import json
import logging

import signac.contrib
from signac.common import six

if six.PY2:
    logging.basicConfig(level=logging.WARNING)
    from tempdir import TemporaryDirectory
else:
    from tempfile import TemporaryDirectory

SIGNAC_ACCESS_MODULE = """import os
import re

import signac.contrib

RE_TXT = ".*a_(?P<a>\d)\.txt"

class Crawler(signac.contrib.RegexFileCrawler):
    tags = {'test1', 'test2'}

Crawler.define(RE_TXT, signac.contrib.formats.TextFile)

def get_crawlers(root):
    return {'main':  Crawler(os.path.join(root, '.'))}
"""

class TestFormat(object):

    def read(self):
        assert 0

    def close(self):
        assert 0

class TestFS(object):
    name = 'inmemorytestgrid'
    files = dict()

    class FileExistsError(IOError):
        pass
    FileNotFoundError = KeyError

    class _Writer(io.BytesIO):

        def __init__(self, cache, file_id):
            self.cache = cache
            self.file_id = file_id

        def close(self):
            self.cache[self.file_id] = self.getvalue()
            super(TestFS._Writer, self).close()

    def __init__(self, _id):
        self._id = _id

    def config(self):
        return {'id': self._id}

    @classmethod
    def from_config(cls, config):
        return TestFS(_id=config['id'])

    def new_file(self, mode='xb', **kwargs):
        _id = kwargs['_id']
        cache = self.files.setdefault(self._id, dict())
        if _id in cache:
            raise self.FileExistsError(_id)
        if mode == 'xb':
            return self._Writer(cache, _id)
        else:
            raise ValueError(mode)

    def get(self, file_id, mode='r'):
        cache = self.files[self._id]
        buf = cache[file_id]
        if mode == 'r':
            return io.StringIO(buf.decode())
        elif mode == 'rb':
            return io.BytesIO(buf)
        else:
            raise ValueError(mode)

class CrawlerBaseTest(unittest.TestCase):

    def setUp(self):
        self._tmp_dir = TemporaryDirectory(prefix='signac_')
        self.addCleanup(self._tmp_dir.cleanup)

    def setup_project(self):
        def fn(name):
            return os.path.join(self._tmp_dir.name, name)
        with open(fn('a_0.txt'), 'w') as file:
            file.write('{"a": 0}')
        with open(fn('a_1.txt'), 'w') as file:
            file.write('{"a": 1}')
        with open(fn('a_0.json'), 'w') as file:
            json.dump(dict(a=0), file)
        with open(fn('a_1.json'), 'w') as file:
            json.dump(dict(a=1), file)
        with open(fn('signac_access.py'), 'w') as module:
            module.write(SIGNAC_ACCESS_MODULE)

    def test_base_crawler(self):
        crawler = signac.contrib.BaseCrawler(root=self._tmp_dir.name)
        self.assertEqual(len(list(crawler.crawl())), 0)
        doc = dict(a=0)
        for doc in crawler.fetch(doc):
            pass
        self.assertEqual(doc, crawler.process(doc, None, None))
        with self.assertRaises(NotImplementedError):
            crawler.docs_from_file(None, None)

    def test_regex_file_crawler_pre_compiled(self):
        self.setup_project()

        class Crawler(signac.contrib.RegexFileCrawler):
            pass

        regex = re.compile(".*a_(?P<a>\d)\.txt")
        Crawler.define(regex, TestFormat)
        crawler = Crawler(root=self._tmp_dir.name)
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            m = regex.match(ffn)
            self.assertIsNotNone(m)
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
        self.assertFalse(no_find)

    def test_regex_file_crawler(self):
        self.setup_project()

        class Crawler(signac.contrib.RegexFileCrawler):
            pass
        pattern = ".*a_(?P<a>\d)\.txt"
        regex = re.compile(pattern)
        Crawler.define(pattern, TestFormat)
        crawler = Crawler(root=self._tmp_dir.name)
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            m = regex.match(ffn)
            self.assertIsNotNone(m)
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
        self.assertFalse(no_find)

    def test_regex_file_crawler_inheritance(self):
        self.setup_project()

        class CrawlerA(signac.contrib.RegexFileCrawler):
            pass

        class CrawlerB(signac.contrib.RegexFileCrawler):
            pass

        CrawlerA.define('a', TestFormat)
        CrawlerB.define('b', TestFormat)
        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)

        class CrawlerC(CrawlerA):
            pass

        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerC.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)
        CrawlerC.define('c', TestFormat)
        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)
        self.assertEqual(len(CrawlerC.definitions), 2)

    def test_json_crawler(self):
        self.setup_project()
        crawler = signac.contrib.JSONCrawler(root=self._tmp_dir.name)
        docs = list(sorted(crawler.crawl(), key=lambda d: d['a']))
        self.assertEqual(len(docs), 2)
        for i, doc in enumerate(docs):
            self.assertEqual(doc['a'], i)
            self.assertIsNone(doc['format'])
        ids = set(doc['_id'] for doc in docs)
        self.assertEqual(len(ids), len(docs))

    def test_master_crawler(self):
        self.setup_project()
        crawler = signac.contrib.MasterCrawler(root=self._tmp_dir.name)
        crawler.tags = {'test1'}
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
            no_files = True
            for file in signac.contrib.fetch(doc):
                no_files = False
                self.assertEqual(signac.contrib.formats.TextFile, type(file))
                file.close()
            self.assertFalse(no_files)
        self.assertFalse(no_find)

    def test_master_crawler_tags(self):
        self.setup_project()
        crawler = signac.contrib.MasterCrawler(root=self._tmp_dir.name)
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = None
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {}
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {'nomatch'}
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {'test1'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test2'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'test2'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test2', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'test2', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))

    def test_custom_mirror(self):
        self.setup_project()
        fs_write = TestFS('custom_filesystem')
        fs_read = TestFS('custom_filesystem')
        fs_bad = TestFS('bad')
        crawler = signac.contrib.MasterCrawler(
            root=self._tmp_dir.name,
            link_local=False,
            mirrors=(fs_write,))
        crawler.tags = {'test1'}
        index = list(crawler.crawl())
        self.assertEqual(len(index), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=(fs_read, ), ignore_linked_mirrors=True):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=(fs_bad, fs_read), ignore_linked_mirrors=True):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
        self.assertEqual(len(check), 2)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=[], ignore_linked_mirrors=True)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=(fs_bad,), ignore_linked_mirrors=True)

    def test_local_filesystem(self):
        self.setup_project()
        fs_root = os.path.join(self._tmp_dir.name, 'local')
        fs_test = signac.contrib.filesystems.LocalFS(fs_root)
        with fs_test.new_file(_id='test123') as file:
            if six.PY2:
                file.write('testfilewrite')
            else:
                file.write('testfilewrite'.encode())
        with self.assertRaises(fs_test.FileExistsError):
            fs_test.new_file(_id='test123')
        with fs_test.get('test123') as file:
            self.assertEqual(file.read(), 'testfilewrite')
        with self.assertRaises(fs_test.FileNotFoundError):
            fs_test.get('badid')
        fs_bad = signac.contrib.filesystems.LocalFS('/bad/path')
        crawler = signac.contrib.MasterCrawler(
            root=self._tmp_dir.name,
            link_local=False,
            mirrors=({'localfs': {'root': fs_root}},))
        crawler.tags = {'test1'}
        index = list(crawler.crawl())
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(doc, sources=(fs_test,)):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=({'localfs': {'root': fs_root}},)):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=[dict(localfs=fs_root)]):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(doc, sources=[]):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(doc, ignore_linked_mirrors=True)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=(fs_bad,), ignore_linked_mirrors=True)

if __name__ == '__main__':
    unittest.main()
<commit_msg>Test RegexFileCrawler without filename pattern definition.<commit_after>import unittest
import os
import io
import re
import json
import logging

import signac.contrib
from signac.common import six

if six.PY2:
    logging.basicConfig(level=logging.WARNING)
    from tempdir import TemporaryDirectory
else:
    from tempfile import TemporaryDirectory

SIGNAC_ACCESS_MODULE = """import os
import re

import signac.contrib

RE_TXT = ".*a_(?P<a>\d)\.txt"

class Crawler(signac.contrib.RegexFileCrawler):
    tags = {'test1', 'test2'}

Crawler.define(RE_TXT, signac.contrib.formats.TextFile)

def get_crawlers(root):
    return {'main':  Crawler(os.path.join(root, '.'))}
"""

class TestFormat(object):

    def read(self):
        assert 0

    def close(self):
        assert 0

class TestFS(object):
    name = 'inmemorytestgrid'
    files = dict()

    class FileExistsError(IOError):
        pass
    FileNotFoundError = KeyError

    class _Writer(io.BytesIO):

        def __init__(self, cache, file_id):
            self.cache = cache
            self.file_id = file_id

        def close(self):
            self.cache[self.file_id] = self.getvalue()
            super(TestFS._Writer, self).close()

    def __init__(self, _id):
        self._id = _id

    def config(self):
        return {'id': self._id}

    @classmethod
    def from_config(cls, config):
        return TestFS(_id=config['id'])

    def new_file(self, mode='xb', **kwargs):
        _id = kwargs['_id']
        cache = self.files.setdefault(self._id, dict())
        if _id in cache:
            raise self.FileExistsError(_id)
        if mode == 'xb':
            return self._Writer(cache, _id)
        else:
            raise ValueError(mode)

    def get(self, file_id, mode='r'):
        cache = self.files[self._id]
        buf = cache[file_id]
        if mode == 'r':
            return io.StringIO(buf.decode())
        elif mode == 'rb':
            return io.BytesIO(buf)
        else:
            raise ValueError(mode)

class CrawlerBaseTest(unittest.TestCase):

    def setUp(self):
        self._tmp_dir = TemporaryDirectory(prefix='signac_')
        self.addCleanup(self._tmp_dir.cleanup)

    def setup_project(self):
        def fn(name):
            return os.path.join(self._tmp_dir.name, name)
        with open(fn('a_0.txt'), 'w') as file:
            file.write('{"a": 0}')
        with open(fn('a_1.txt'), 'w') as file:
            file.write('{"a": 1}')
        with open(fn('a_0.json'), 'w') as file:
            json.dump(dict(a=0), file)
        with open(fn('a_1.json'), 'w') as file:
            json.dump(dict(a=1), file)
        with open(fn('signac_access.py'), 'w') as module:
            module.write(SIGNAC_ACCESS_MODULE)

    def test_base_crawler(self):
        crawler = signac.contrib.BaseCrawler(root=self._tmp_dir.name)
        self.assertEqual(len(list(crawler.crawl())), 0)
        doc = dict(a=0)
        for doc in crawler.fetch(doc):
            pass
        self.assertEqual(doc, crawler.process(doc, None, None))
        with self.assertRaises(NotImplementedError):
            crawler.docs_from_file(None, None)

    def test_regex_file_crawler_pre_compiled(self):
        self.setup_project()

        class Crawler(signac.contrib.RegexFileCrawler):
            pass

        regex = re.compile(".*a_(?P<a>\d)\.txt")
        Crawler.define(regex, TestFormat)
        crawler = Crawler(root=self._tmp_dir.name)
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            m = regex.match(ffn)
            self.assertIsNotNone(m)
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
        self.assertFalse(no_find)

    def test_regex_file_crawler(self):
        self.setup_project()

        class Crawler(signac.contrib.RegexFileCrawler):
            pass

        # First test without pattern
        crawler = Crawler(root=self._tmp_dir.name)
        self.assertEqual(len(list(crawler.crawl())), 0)

        # Now with pattern
        pattern = ".*a_(?P<a>\d)\.txt"
        regex = re.compile(pattern)
        Crawler.define(pattern, TestFormat)
        crawler = Crawler(root=self._tmp_dir.name)
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            m = regex.match(ffn)
            self.assertIsNotNone(m)
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
        self.assertFalse(no_find)

    def test_regex_file_crawler_inheritance(self):
        self.setup_project()

        class CrawlerA(signac.contrib.RegexFileCrawler):
            pass

        class CrawlerB(signac.contrib.RegexFileCrawler):
            pass

        CrawlerA.define('a', TestFormat)
        CrawlerB.define('b', TestFormat)
        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)

        class CrawlerC(CrawlerA):
            pass

        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerC.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)
        CrawlerC.define('c', TestFormat)
        self.assertEqual(len(CrawlerA.definitions), 1)
        self.assertEqual(len(CrawlerB.definitions), 1)
        self.assertEqual(len(CrawlerC.definitions), 2)

    def test_json_crawler(self):
        self.setup_project()
        crawler = signac.contrib.JSONCrawler(root=self._tmp_dir.name)
        docs = list(sorted(crawler.crawl(), key=lambda d: d['a']))
        self.assertEqual(len(docs), 2)
        for i, doc in enumerate(docs):
            self.assertEqual(doc['a'], i)
            self.assertIsNone(doc['format'])
        ids = set(doc['_id'] for doc in docs)
        self.assertEqual(len(ids), len(docs))

    def test_master_crawler(self):
        self.setup_project()
        crawler = signac.contrib.MasterCrawler(root=self._tmp_dir.name)
        crawler.tags = {'test1'}
        no_find = True
        for doc in crawler.crawl():
            no_find = False
            ffn = os.path.join(doc['root'], doc['filename'])
            self.assertTrue(os.path.isfile(ffn))
            with open(ffn) as file:
                doc2 = json.load(file)
                self.assertEqual(doc2['a'], doc['a'])
            no_files = True
            for file in signac.contrib.fetch(doc):
                no_files = False
                self.assertEqual(signac.contrib.formats.TextFile, type(file))
                file.close()
            self.assertFalse(no_files)
        self.assertFalse(no_find)

    def test_master_crawler_tags(self):
        self.setup_project()
        crawler = signac.contrib.MasterCrawler(root=self._tmp_dir.name)
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = None
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {}
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {'nomatch'}
        self.assertEqual(0, len(list(crawler.crawl())))
        crawler.tags = {'test1'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test2'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'test2'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test2', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))
        crawler.tags = {'test1', 'test2', 'bs'}
        self.assertEqual(2, len(list(crawler.crawl())))

    def test_custom_mirror(self):
        self.setup_project()
        fs_write = TestFS('custom_filesystem')
        fs_read = TestFS('custom_filesystem')
        fs_bad = TestFS('bad')
        crawler = signac.contrib.MasterCrawler(
            root=self._tmp_dir.name,
            link_local=False,
            mirrors=(fs_write,))
        crawler.tags = {'test1'}
        index = list(crawler.crawl())
        self.assertEqual(len(index), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=(fs_read, ), ignore_linked_mirrors=True):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=(fs_bad, fs_read), ignore_linked_mirrors=True):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
        self.assertEqual(len(check), 2)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=[], ignore_linked_mirrors=True)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=(fs_bad,), ignore_linked_mirrors=True)

    def test_local_filesystem(self):
        self.setup_project()
        fs_root = os.path.join(self._tmp_dir.name, 'local')
        fs_test = signac.contrib.filesystems.LocalFS(fs_root)
        with fs_test.new_file(_id='test123') as file:
            if six.PY2:
                file.write('testfilewrite')
            else:
                file.write('testfilewrite'.encode())
        with self.assertRaises(fs_test.FileExistsError):
            fs_test.new_file(_id='test123')
        with fs_test.get('test123') as file:
            self.assertEqual(file.read(), 'testfilewrite')
        with self.assertRaises(fs_test.FileNotFoundError):
            fs_test.get('badid')
        fs_bad = signac.contrib.filesystems.LocalFS('/bad/path')
        crawler = signac.contrib.MasterCrawler(
            root=self._tmp_dir.name,
            link_local=False,
            mirrors=({'localfs': {'root': fs_root}},))
        crawler.tags = {'test1'}
        index = list(crawler.crawl())
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(doc, sources=(fs_test,)):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=({'localfs': {'root': fs_root}},)):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(
                    doc, sources=[dict(localfs=fs_root)]):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            for file in signac.contrib.fetch(doc, sources=[]):
                m = json.load(file)
                self.assertTrue('a' in m)
                check.append(m)
                file.close()
        self.assertEqual(len(check), 2)
        check = list()
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(doc, ignore_linked_mirrors=True)
        for doc in index:
            with self.assertRaises(IOError):
                signac.contrib.fetch_one(
                    doc, sources=(fs_bad,), ignore_linked_mirrors=True)

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
import re
from mecabwrap.domecab import do_mecab
from mecabwrap.domecab import do_mecab_vec
from mecabwrap.domecab import do_mecab_iter

class TestDomecab(unittest.TestCase):
    def test_wakati(self):
        out = do_mecab(u'すもももももももものうち', '-Owakati')
        self.assertEqual(out.strip(), u'すもも も もも も もも の うち')

    def test_default(self):
        out = do_mecab(u'メロンパンを食べる')
        lbs = re.findall(r'\n', out)
        self.assertEqual(len(lbs), 5)

        words = re.findall(r'[^\t\n]+\t', out)
        words = [w[:-1] for w in words]
        self.assertEqual(words, [u'メロン', u'パン', u'を', u'食べる'])

class TestDomecabVec(unittest.TestCase):
    def test_vec(self):
        ins = [u'春はあけぼの', u'やうやう白くなりゆく山際']
        out = do_mecab_vec(ins, outpath=None)
        lbs = re.findall(r'\n', out)
        self.assertEqual(len(lbs), 10)

        words = re.findall(r'[^\t\n]+\t', out)
        words = [w[:-1] for w in words]
        self.assertEqual(
            words,
            [u'春', u'は', u'あけぼの', u'やうやう',
             u'白く', u'なり', u'ゆく', u'山際']
        )

        lines = out.split('\n')
        lines = [line.strip() for line in lines]
        self.assertEqual(lines[3], 'EOS')
        self.assertEqual(lines[9], 'EOS')

class TestDomecabIter(unittest.TestCase):
    def test_iter(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, byline=False):
            ct += 1
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-Owakati', byline=True):
            ct += 1
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-Owakati', byline=False):
            ct += 1
        self.assertEqual(ct, 1)

    def test_iter_Eopt(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, u'-EEND\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], u'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-E', u'END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], u'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, u'--eos-format=END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], u'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '--eos-format', u'END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], u'END')
        self.assertEqual(ct, 2)

    def test_iter_Eopt_unicode(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, u'-Eおしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-E', u'おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, u'--eos-format=おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '--eos-format', u'おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

class TestMultipleOptions(unittest.TestCase):
    def test_multiple_options(self):
        out = do_mecab(u"すもももももももものうち", '-Bbegin\n', '-Eend\n')
        out = out.strip()
        out = re.split(r'(\r\n)|(\n)|(\r)', out)
        self.assertEqual(out[0], 'begin')
        self.assertEqual(out[-1], 'end')

if __name__ == '__main__':
    unittest.main()

<commit_msg>try encoding fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import unittest
import re
from mecabwrap.domecab import do_mecab
from mecabwrap.domecab import do_mecab_vec
from mecabwrap.domecab import do_mecab_iter

class TestDomecab(unittest.TestCase):
    def test_wakati(self):
        out = do_mecab(u'すもももももももものうち', '-Owakati')
        self.assertEqual(out.strip(), u'すもも も もも も もも の うち')

    def test_default(self):
        out = do_mecab(u'メロンパンを食べる')
        lbs = re.findall(r'\n', out)
        self.assertEqual(len(lbs), 5)

        words = re.findall(r'[^\t\n]+\t', out)
        words = [w[:-1] for w in words]
        self.assertEqual(words, [u'メロン', u'パン', u'を', u'食べる'])

class TestDomecabVec(unittest.TestCase):
    def test_vec(self):
        ins = [u'春はあけぼの', u'やうやう白くなりゆく山際']
        out = do_mecab_vec(ins, outpath=None)
        lbs = re.findall(r'\n', out)
        self.assertEqual(len(lbs), 10)

        words = re.findall(r'[^\t\n]+\t', out)
        words = [w[:-1] for w in words]
        self.assertEqual(
            words,
            [u'春', u'は', u'あけぼの', u'やうやう',
             u'白く', u'なり', u'ゆく', u'山際']
        )

        lines = out.split('\n')
        lines = [line.strip() for line in lines]
        self.assertEqual(lines[3], 'EOS')
        self.assertEqual(lines[9], 'EOS')

class TestDomecabIter(unittest.TestCase):
    def test_iter(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, byline=False):
            ct += 1
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-Owakati', byline=True):
            ct += 1
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-Owakati', byline=False):
            ct += 1
        self.assertEqual(ct, 1)

    def test_iter_Eopt(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, '-EEND\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], 'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-E', 'END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], 'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '--eos-format=END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], 'END')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '--eos-format', 'END\n', byline=False):
            ct += 1
            self.assertEqual(line[-3:], 'END')
        self.assertEqual(ct, 2)

    def test_iter_Eopt_unicode(self):
        ins = [u'となりの客はよく柿食う客だ', u'バスガス爆発']
        ct = 0
        for line in do_mecab_iter(ins, u'-Eおしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '-E', u'おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, u'--eos-format=おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

        ct = 0
        for line in do_mecab_iter(ins, '--eos-format', u'おしまい\n', byline=False):
            ct += 1
            self.assertEqual(line[-4:], u'おしまい')
        self.assertEqual(ct, 2)

class TestMultipleOptions(unittest.TestCase):
    def test_multiple_options(self):
        out = do_mecab(u"すもももももももものうち", '-Bbegin\n', '-Eend\n')
        out = out.strip()
        out = re.split(r'(\r\n)|(\n)|(\r)', out)
        self.assertEqual(out[0], 'begin')
        self.assertEqual(out[-1], 'end')

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

"""Shared constant values."""

__all__ = [
    'DEFAULT_CHARSET',
    'DEFAULT_ENCTYPE',
    'TASK_STATUSES',
]

DEFAULT_CHARSET = u'utf8'
DEFAULT_ENCTYPE = u'application/x-www-form-urlencoded'

PROXY_HEADER_PREFIX = u'Torque-Passthrough-'

TASK_STATUSES = {
    'completed': u'COMPLETED',
    'failed': u'FAILED',
    'pending': u'PENDING',
}
<commit_msg>model: lose constants __all__ list.<commit_after># -*- coding: utf-8 -*-

"""Shared constant values."""

DEFAULT_CHARSET = u'utf8'
DEFAULT_ENCTYPE = u'application/x-www-form-urlencoded'
PROXY_HEADER_PREFIX = u'Torque-Passthrough-'
TASK_STATUSES = {
    'completed': u'COMPLETED',
    'failed': u'FAILED',
    'pending': u'PENDING',
}
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals
from thinc.neural import Model
import pytest
import numpy

from ..._ml import chain, Tok2Vec, doc2feats
from ...vocab import Vocab
from ...pipeline import TokenVectorEncoder
from ...syntax.arc_eager import ArcEager
from ...syntax.nn_parser import Parser
from ...tokens.doc import Doc
from ...gold import GoldParse

@pytest.fixture
def vocab():
    return Vocab()

@pytest.fixture
def arc_eager(vocab):
    actions = ArcEager.get_actions(left_labels=['L'], right_labels=['R'])
    return ArcEager(vocab.strings, actions)

@pytest.fixture
def tok2vec():
    return Tok2Vec(8, 100, preprocess=doc2feats())

@pytest.fixture
def parser(vocab, arc_eager):
    return Parser(vocab, moves=arc_eager, model=None)

@pytest.fixture
def model(arc_eager, tok2vec):
    return Parser.Model(arc_eager.n_moves, token_vector_width=tok2vec.nO)[0]

@pytest.fixture
def doc(vocab):
    return Doc(vocab, words=['a', 'b', 'c'])

@pytest.fixture
def gold(doc):
    return GoldParse(doc, heads=[1, 1, 1], deps=['L', 'ROOT', 'R'])

def test_can_init_nn_parser(parser):
    assert parser.model is None

def test_build_model(parser):
    parser.model = Parser.Model(parser.moves.n_moves)[0]
    assert parser.model is not None

def test_predict_doc(parser, tok2vec, model, doc):
    doc.tensor = tok2vec([doc])[0]
    parser.model = model
    parser(doc)

def test_update_doc(parser, tok2vec, model, doc, gold):
    parser.model = model
    tokvecs, bp_tokvecs = tok2vec.begin_update([doc])
    d_tokvecs = parser.update(([doc], tokvecs), [gold])
    assert d_tokvecs[0].shape == tokvecs[0].shape
    def optimize(weights, gradient, key=None):
        weights -= 0.001 * gradient
    bp_tokvecs(d_tokvecs, sgd=optimize)
    assert d_tokvecs[0].sum() == 0.

def test_predict_doc_beam(parser, tok2vec, model, doc):
    doc.tensor = tok2vec([doc])[0]
    parser.model = model
    parser(doc, beam_width=32, beam_density=0.001)
    for word in doc:
        print(word.text, word.head, word.dep_)
<commit_msg>Add test for beam update<commit_after># coding: utf8
from __future__ import unicode_literals
from thinc.neural import Model
import pytest
import numpy

from ..._ml import chain, Tok2Vec, doc2feats
from ...vocab import Vocab
from ...pipeline import TokenVectorEncoder
from ...syntax.arc_eager import ArcEager
from ...syntax.nn_parser import Parser
from ...tokens.doc import Doc
from ...gold import GoldParse

@pytest.fixture
def vocab():
    return Vocab()

@pytest.fixture
def arc_eager(vocab):
    actions = ArcEager.get_actions(left_labels=['L'], right_labels=['R'])
    return ArcEager(vocab.strings, actions)

@pytest.fixture
def tok2vec():
    return Tok2Vec(8, 100, preprocess=doc2feats())

@pytest.fixture
def parser(vocab, arc_eager):
    return Parser(vocab, moves=arc_eager, model=None)

@pytest.fixture
def model(arc_eager, tok2vec):
    return Parser.Model(arc_eager.n_moves, token_vector_width=tok2vec.nO)[0]

@pytest.fixture
def doc(vocab):
    return Doc(vocab, words=['a', 'b', 'c'])

@pytest.fixture
def gold(doc):
    return GoldParse(doc, heads=[1, 1, 1], deps=['L', 'ROOT', 'R'])

def test_can_init_nn_parser(parser):
    assert parser.model is None

def test_build_model(parser):
    parser.model = Parser.Model(parser.moves.n_moves)[0]
    assert parser.model is not None

def test_predict_doc(parser, tok2vec, model, doc):
    doc.tensor = tok2vec([doc])[0]
    parser.model = model
    parser(doc)

def test_update_doc(parser, tok2vec, model, doc, gold):
    parser.model = model
    tokvecs, bp_tokvecs = tok2vec.begin_update([doc])
    d_tokvecs = parser.update(([doc], tokvecs), [gold])
    assert d_tokvecs[0].shape == tokvecs[0].shape
    def optimize(weights, gradient, key=None):
        weights -= 0.001 * gradient
    bp_tokvecs(d_tokvecs, sgd=optimize)
    assert d_tokvecs[0].sum() == 0.

def test_predict_doc_beam(parser, tok2vec, model, doc):
    doc.tensor = tok2vec([doc])[0]
    parser.model = model
    parser(doc, beam_width=32, beam_density=0.001)
    for word in doc:
        print(word.text, word.head, word.dep_)

def test_update_doc_beam(parser, tok2vec, model, doc, gold):
    parser.model = model
    tokvecs, bp_tokvecs = tok2vec.begin_update([doc])
    d_tokvecs = parser.update_beam(([doc], tokvecs), [gold])
    assert d_tokvecs[0].shape == tokvecs[0].shape
    def optimize(weights, gradient, key=None):
        weights -= 0.001 * gradient
    bp_tokvecs(d_tokvecs, sgd=optimize)
    assert d_tokvecs[0].sum() == 0.

<|endoftext|>


<commit_before>"""Simple test."""

import os
import json
import time
import signal
import requests
import websocket
import subprocess

DAEMON = None
LOGLEVEL = 'DEBUG'

def setup_module():
    global DAEMON

    # call os.setsid so that all subprocesses terminate when the
    # main process receives SIGTERM
    DAEMON = subprocess.Popen(['databench', '--with-coverage',
                               '--log={}'.format(LOGLEVEL)],
                              close_fds=True,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              preexec_fn=os.setsid)
    time.sleep(1)

def teardown_module():
    global DAEMON

    # simply DAEMON.terminate() would only terminate the main process,
    # but the nested processes also need to be terminated
    #
    # SIGUSR1 does not exist on Windows
    if hasattr(signal, 'SIGUSR1'):
        os.killpg(DAEMON.pid, signal.SIGUSR1)
    else:
        os.killpg(DAEMON.pid, signal.SIGTERM)
    DAEMON.wait()

"""

Testing a default Python analysis
---------------------------------
"""

def test_get_dummypi():
    r = requests.get('http://127.0.0.1:5000/dummypi/')
    assert r.status_code == 200

def test_ws_dummypi():
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/dummypi/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"run", "load":{"__action_id":123}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__action'
    assert r['load']['id'] == 123
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'log'
    assert 'inside' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'pi' in r['load']
    ws.close()

def test_multiple_ws_dummypi():
    # websocket.enableTrace(True)
    wss = [websocket.create_connection('ws://127.0.0.1:5000/dummypi/ws')
           for _ in range(4)]
    for ws in wss:
        ws.send('{"__connect": null}')
        r = json.loads(ws.recv())
        print(r)
        assert r['signal'] == '__connect'
        assert 'analysis_id' in r['load']
    for ws in wss:
        ws.close()

"""

Python Language Kernel tests
----------------------------
"""

def test_get_dummypi_py():
    r = requests.get('http://127.0.0.1:5000/dummypi_py/')
    assert r.status_code == 200

def test_multiple_ws_dummypi_py():
    # websocket.enableTrace(True)
    wss = [websocket.create_connection('ws://127.0.0.1:5000/dummypi_py/ws')
           for _ in range(4)]
    for ws in wss:
        ws.send('{"__connect": null}')
        r = json.loads(ws.recv())
        print(r)
        assert r['signal'] == '__connect'
        assert 'analysis_id' in r['load']
    for ws in wss:
        ws.close()

def test_ws_dummypi_py():
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/dummypi_py/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    time.sleep(1)
    ws.send('{"signal":"run", "load":{"__action_id":123}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__action'
    assert r['load']['id'] == 123
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'log'
    assert 'inside' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'pi' in r['load']
    ws.close()

"""

Function Argument Tests
-----------------------
"""

def _fn_call(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    time.sleep(1)
    ws.send('{"signal":"test_fn", "load": 1}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 100
    ws.close()

def _fn_call_array(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    time.sleep(1)
    ws.send('{"signal":"test_fn", "load": [1, 2]}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 2
    ws.close()

def _fn_call_dict(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    time.sleep(1)
    ws.send('{"signal":"test_fn", "load": '
            '{"first_param": 1, "second_param": 2}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 2
    ws.close()

def test_fn_call_dummypi():
    _fn_call()

def test_fn_call_dummypi_array():
    _fn_call_array()

def test_fn_call_dummypi_dict():
    _fn_call_dict()

def test_fn_call_dummypi_py():
    _fn_call(name='dummypi_py')

def test_fn_call_dummypi_py_array():
    _fn_call_array(name='dummypi_py')

def test_fn_call_dummypi_py_dict():
    _fn_call_dict(name='dummypi_py')

if __name__ == '__main__':
    LOGLEVEL = 'DEBUG'
    setup_module()
    test_ws_dummypi_py()
    teardown_module()
<commit_msg>remove sleeps from tests<commit_after>"""Simple test."""

import os
import json
import time
import signal
import requests
import websocket
import subprocess

DAEMON = None
LOGLEVEL = 'DEBUG'

def setup_module():
    global DAEMON

    # call os.setsid so that all subprocesses terminate when the
    # main process receives SIGTERM
    DAEMON = subprocess.Popen(['databench', '--with-coverage',
                               '--log={}'.format(LOGLEVEL)],
                              close_fds=True,
                              stdin=subprocess.PIPE,
                              stdout=subprocess.PIPE,
                              preexec_fn=os.setsid)
    time.sleep(1)

def teardown_module():
    global DAEMON

    # simply DAEMON.terminate() would only terminate the main process,
    # but the nested processes also need to be terminated
    #
    # SIGUSR1 does not exist on Windows
    if hasattr(signal, 'SIGUSR1'):
        os.killpg(DAEMON.pid, signal.SIGUSR1)
    else:
        os.killpg(DAEMON.pid, signal.SIGTERM)
    DAEMON.wait()

"""

Testing a default Python analysis
---------------------------------
"""

def test_get_dummypi():
    r = requests.get('http://127.0.0.1:5000/dummypi/')
    assert r.status_code == 200

def test_ws_dummypi():
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/dummypi/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"run", "load":{"__action_id":123}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__action'
    assert r['load']['id'] == 123
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'log'
    assert 'inside' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'pi' in r['load']
    ws.close()

def test_multiple_ws_dummypi():
    # websocket.enableTrace(True)
    wss = [websocket.create_connection('ws://127.0.0.1:5000/dummypi/ws')
           for _ in range(4)]
    for ws in wss:
        ws.send('{"__connect": null}')
        r = json.loads(ws.recv())
        print(r)
        assert r['signal'] == '__connect'
        assert 'analysis_id' in r['load']
    for ws in wss:
        ws.close()

"""

Python Language Kernel tests
----------------------------
"""

def test_get_dummypi_py():
    r = requests.get('http://127.0.0.1:5000/dummypi_py/')
    assert r.status_code == 200

def test_multiple_ws_dummypi_py():
    # websocket.enableTrace(True)
    wss = [websocket.create_connection('ws://127.0.0.1:5000/dummypi_py/ws')
           for _ in range(4)]
    for ws in wss:
        ws.send('{"__connect": null}')
        r = json.loads(ws.recv())
        print(r)
        assert r['signal'] == '__connect'
        assert 'analysis_id' in r['load']
    for ws in wss:
        ws.close()

def test_ws_dummypi_py():
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/dummypi_py/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"run", "load":{"__action_id":123}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__action'
    assert r['load']['id'] == 123
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'log'
    assert 'inside' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'pi' in r['load']
    ws.close()

"""

Function Argument Tests
-----------------------
"""

def _fn_call(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"test_fn", "load": 1}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 100
    ws.close()

def _fn_call_array(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"test_fn", "load": [1, 2]}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 2
    ws.close()

def _fn_call_dict(name='dummypi'):
    # websocket.enableTrace(True)
    ws = websocket.create_connection('ws://127.0.0.1:5000/'+name+'/ws')
    ws.send('{"__connect": null}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == '__connect'
    assert 'analysis_id' in r['load']
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'data'
    assert 'samples' in r['load']
    ws.send('{"signal":"test_fn", "load": '
            '{"first_param": 1, "second_param": 2}}')
    r = json.loads(ws.recv())
    print(r)
    assert r['signal'] == 'test_fn'
    assert r['load']['first_param'] == 1
    assert r['load']['second_param'] == 2
    ws.close()

def test_fn_call_dummypi():
    _fn_call()

def test_fn_call_dummypi_array():
    _fn_call_array()

def test_fn_call_dummypi_dict():
    _fn_call_dict()

def test_fn_call_dummypi_py():
    _fn_call(name='dummypi_py')

def test_fn_call_dummypi_py_array():
    _fn_call_array(name='dummypi_py')

def test_fn_call_dummypi_py_dict():
    _fn_call_dict(name='dummypi_py')

if __name__ == '__main__':
    LOGLEVEL = 'DEBUG'
    setup_module()
    test_ws_dummypi_py()
    teardown_module()
<|endoftext|>


<commit_before># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
Tests for ``txkube._model``.
"""

from json import loads, dumps

from zope.interface import implementer
from zope.interface.verify import verifyObject

import attr

from pyrsistent import (
    InvariantException,
    freeze,
)

from testtools.matchers import (
    Equals, MatchesStructure, Not, Is, Contains, ContainsAll, Raises, raises,
    IsInstance,
)
from testtools.twistedsupport import succeeded

from hypothesis import HealthCheck, settings, given, assume
from hypothesis.strategies import sampled_from, choices

from twisted.python.failure import Failure
from twisted.internet.defer import gatherResults
from twisted.web.iweb import IResponse
from twisted.web.http_headers import Headers
from twisted.web.client import ResponseDone

from ..testing import TestCase
from ..testing.matchers import (
    PClassEquals,
    EqualElements,
    MappingEquals,
    raises_exception,
)
from ..testing.strategies import (
    iobjects,
    namespacelists,
    objectcollections,
)

from .. import (
    UnrecognizedVersion, UnrecognizedKind,
    KubernetesError,
    IObject,
    v1_5_model, openapi_to_data_model,
)

from .._model import set_if_none

def models():
    return sampled_from([v1_5_model])

class SerializationTests(TestCase):
    """
    Tests for ``iobject_to_raw`` and ``iobject_from_raw``.
    """
    def test_v1_apiVersion(self):
        """
        Objects from ``v1`` serialize with an *apiVersion* of ``u"v1"``.
        """
        model = v1_5_model

        obj = model.v1.ComponentStatus()
        raw = model.iobject_to_raw(obj)
        self.expectThat(
            raw[u"apiVersion"],
            Equals(u"v1"),
        )
        self.expectThat(
            model.iobject_from_raw(raw),
            IsInstance(model.v1.ComponentStatus),
        )

    def test_v1beta1_apiVersion(self):
        """
        Objects from ``v1beta1`` serialize with an *apiVersion* of
        ``u"extensions/v1beta1"``.
        """
        model = v1_5_model

        obj = model.v1beta1.CertificateSigningRequest()
        raw = model.iobject_to_raw(obj)
        self.expectThat(
            raw[u"apiVersion"],
            Equals(u"extensions/v1beta1"),
        )
        self.expectThat(
            model.iobject_from_raw(raw),
            IsInstance(model.v1beta1.CertificateSigningRequest),
        )

class IObjectTests(TestCase):
    """
    Tests for ``IObject``.
    """
    @given(obj=iobjects())
    def test_interface(self, obj):
        """
        The object provides ``IObject``.
        """
        verifyObject(IObject, obj)

    @given(models())
    def test_constant_attributes(self, model):
        """
        The ``apiVersion`` and ``kind`` attributes reflect the Kubernetes object
        apiVersion and kind fields.
        """
        p = model.v1.Pod()
        self.expectThat(p.apiVersion, Equals(u"v1"))
        self.expectThat(p.kind, Equals(u"Pod"))

        pl = model.v1.PodList()
        self.expectThat(pl.apiVersion, Equals(u"v1"))
        self.expectThat(pl.kind, Equals(u"PodList"))

        d = model.v1beta1.Deployment()
        self.expectThat(d.apiVersion, Equals(u"v1beta1"))
        self.expectThat(d.kind, Equals(u"Deployment"))

        dl = model.v1beta1.DeploymentList()
        self.expectThat(dl.apiVersion, Equals(u"v1beta1"))
        self.expectThat(dl.kind, Equals(u"DeploymentList"))

    @given(obj=iobjects())
    def test_serialization_roundtrip(self, obj):
        """
        An ``IObject`` provider can be round-trip through JSON using
        ``iobject_to_raw`` and ``iobject_from_raw``.
        """
        model = v1_5_model

        marshalled = model.iobject_to_raw(obj)

        # Every IObject has these marshalled fields - and when looking at the
        # marshalled form, they're necessary to figure out the
        # schema/definition for the data.  We can't say anything in general
        # about the *values* (because of things like "extensions/v1beta1") but
        # we can at least assert the keys are present.
        self.expectThat(marshalled, ContainsAll([u"kind", u"apiVersion"]))

        # We should be able to unmarshal the data back to the same model
        # object as we started with.
        reloaded = model.iobject_from_raw(marshalled)
        self.expectThat(obj, PClassEquals(reloaded))

        # And, to be extra sure (ruling out any weird Python object
        # semantic hijinx), that that reconstituted object should marshal
        # back to exactly the same simplified object graph.
        remarshalled = model.iobject_to_raw(reloaded)
        self.expectThat(marshalled, MappingEquals(remarshalled))

        # Also, the marshalled form must be JSON compatible.
        serialized = dumps(marshalled)
        deserialized = loads(serialized)
        self.expectThat(marshalled, MappingEquals(deserialized))

    @given(objectcollections())
    def test_empty_collection(self, collection):
        """
        The ``items`` of a collection can be made empty in a couple different
        ways.
        """
        self.expectThat(collection.set(items=None).items, Equals([]))
        self.expectThat(collection.set(items=[]).items, Equals([]))

    @given(collection=namespacelists(), choose=choices())
    def test_unique_contents(self, collection, choose):
        """
        A collection type cannot contain more than one object with a particular
        namespace / name pair.
        """
        assume(len(collection.items) > 0)
        item = choose(collection.items)
        self.expectThat(
            lambda: collection.add(item),
            raises_exception(InvariantException),
        )

    def test_unknown_version(self):
        """
        ``iobject_from_raw`` raises ``UnrecognizedVersion`` if it does not
        recognize the *apiVersion* in the given data.
        """
        model = v1_5_model

        obj = {
            u"apiVersion": u"invalid.example.txkube",
            u"kind": u"Service",
        }
        self.assertThat(
            lambda: model.iobject_from_raw(obj),
            raises(UnrecognizedVersion(obj[u"apiVersion"], obj)),
        )

    def test_unknown_kind(self):
        """
        ``iobject_from_raw`` raises ``UnrecognizedKind`` if it does not recognize
        the *kind* in the given data.
        """
        model = v1_5_model

        obj = {
            u"apiVersion": u"v1",
            u"kind": u"SomethingFictional",
        }
        self.assertThat(
            lambda: model.iobject_from_raw(obj),
            raises(UnrecognizedKind(u"v1", u"SomethingFictional", obj)),
        )

class NamespaceTests(TestCase):
    """
    Other tests for ``Namespace``.
    """
    @given(models())
    def test_default(self, model):
        """
        ``Namespace.default`` returns the *default* namespace.
        """
        self.assertThat(
            model.v1.Namespace.default(),
            MatchesStructure(
                metadata=MatchesStructure(
                    name=Equals(u"default"),
                ),
            ),
        )

    @given(models())
    def test_fill_defaults(self, model):
        """
        ``Namespace.fill_defaults`` returns a ``Namespace`` with *uid* metadata
        and an active *status*.
        """
        # If they are not set already, a uid is generated and put into the
        # metadata and the status is set to active.
        sparse = model.v1.Namespace(metadata=model.v1.ObjectMeta(name=u"foo"))
        filled = sparse.fill_defaults()
        self.expectThat(
            filled,
            MatchesStructure(
                metadata=MatchesStructure(
                    uid=Not(Is(None)),
                ),
                status=Equals(model.v1.NamespaceStatus.active()),
            ),
        )

class NamespaceListTests(TestCase):
    """
    Tests for ``NamespaceList``.
    """
    @settings(suppress_health_check=[HealthCheck.exception_in_generation])
    @given(collection=namespacelists(), choose=choices())
    def test_remove(self, collection, choose):
        """
        ``NamespaceList.remove`` creates a new ``NamespaceList`` which does not
        have the given item.
        """
        assume(len(collection.items) > 0)
        item = choose(collection.items)
        removed = collection.remove(item)
        self.assertThat(removed.items, Not(Contains(item)))

    @given(collection=namespacelists(), choose=choices())
    def test_item_by_name(self, collection, choose):
        """
        ``NamespaceList.item_by_name`` returns the ``Namespace`` with the matching
        name.
        """
        assume(len(collection.items) > 0)
        for item in collection.items:
            self.expectThat(collection.item_by_name(item.metadata.name), Is(item))

        item = choose(collection.items)
        collection = collection.remove(item)
        self.expectThat(
            lambda: collection.item_by_name(item.metadata.name),
            raises(KeyError(item.metadata.name)),
        )

    @given(collection=namespacelists(), choose=choices())
    def test_no_duplicates(self, collection, choose):
        assume(len(collection.items) > 0)
        self.expectThat(
            lambda: collection.add(choose(collection.items)),
            raises_exception(InvariantException),
        )

    @given(collection=namespacelists())
    def test_constant_attributes(self, collection):
        self.expectThat(collection.kind, Equals(u"NamespaceList"))
        self.expectThat(collection.apiVersion, Equals(u"v1"))

class SetIfNoneTests(TestCase):
    """
    Tests for ``set_if_none``.
    """
    def test_none(self):
        """
        If the value for transformation is ``None``, the result contains the new
        value instead.
        """
        structure = freeze({u"foo": None})
        transformed = structure.transform([u"foo"], set_if_none(u"bar"))
        self.assertThat(transformed[u"foo"], Equals(u"bar"))

    def test_not_none(self):
        """
        If the value for transformation is not ``None``, the result contains the
        original value.
        """
        structure = freeze({u"foo": u"baz"})
        transformed = structure.transform([u"foo"], set_if_none(u"bar"))
        self.assertThat(transformed[u"foo"], Equals(u"baz"))

@implementer(IResponse)
@attr.s
class MemoryResponse(object):
    version = attr.ib()
    code = attr.ib()
    phrase = attr.ib()
    headers = attr.ib()
    length = attr.ib()

    request = attr.ib()
    previousResponse = attr.ib()

    _body = attr.ib()

    def deliverBody(self, protocol):
        protocol.makeConnection(None)
        protocol.dataReceived(self._body)
        protocol.connectionLost(Failure(ResponseDone()))

class KubernetesErrorTests(TestCase):
    """
    Tests for ``KubernetesError``.
    """
    def test_from_response(self):
        """
        ``from_response`` returns the same value as ``from_model_and_response``
        when called with the v1.5 model.
        """
        def response():
            body = dumps(v1_5_model.iobject_to_raw(v1_5_model.v1.Status()))
            return MemoryResponse(
                version=(b"HTTP", 1, 1),
                code=200,
                phrase=b"OK",
                headers=Headers(),
                length=len(body),
                request=None,
                previousResponse=None,
                body=body,
            )

        ds = gatherResults([
            KubernetesError.from_response(response()),
            KubernetesError.from_model_and_response(v1_5_model, response()),
        ])
        self.assertThat(
            ds,
            succeeded(EqualElements()),
        )

class Extra15DataModelTests(TestCase):
    """
    Tests for handling of certain Kubernetes 1.5 Swagger specifications.
    """
    def test_status(self):
        """
        If the Kubernetes-reported Swagger specification is missing the Status
        definitions, ``openapi_to_data_model`` returns a data model that
        defines them anyway.
        """
        openapi = v1_5_model.spec.to_document()
        openapi[u"definitions"].clear()
        model = openapi_to_data_model(openapi)
        model.v1.Status
        model.v1.StatusCause
        model.v1.StatusDetails
        model.v1.ListMeta
<commit_msg>unused import<commit_after># Copyright Least Authority Enterprises.
# See LICENSE for details.

"""
Tests for ``txkube._model``.
"""

from json import loads, dumps

from zope.interface import implementer
from zope.interface.verify import verifyObject

import attr

from pyrsistent import (
    InvariantException,
    freeze,
)

from testtools.matchers import (
    Equals, MatchesStructure, Not, Is, Contains, ContainsAll, raises,
    IsInstance,
)
from testtools.twistedsupport import succeeded

from hypothesis import HealthCheck, settings, given, assume
from hypothesis.strategies import sampled_from, choices

from twisted.python.failure import Failure
from twisted.internet.defer import gatherResults
from twisted.web.iweb import IResponse
from twisted.web.http_headers import Headers
from twisted.web.client import ResponseDone

from ..testing import TestCase
from ..testing.matchers import (
    PClassEquals,
    EqualElements,
    MappingEquals,
    raises_exception,
)
from ..testing.strategies import (
    iobjects,
    namespacelists,
    objectcollections,
)

from .. import (
    UnrecognizedVersion, UnrecognizedKind,
    KubernetesError,
    IObject,
    v1_5_model, openapi_to_data_model,
)

from .._model import set_if_none

def models():
    return sampled_from([v1_5_model])

class SerializationTests(TestCase):
    """
    Tests for ``iobject_to_raw`` and ``iobject_from_raw``.
    """
    def test_v1_apiVersion(self):
        """
        Objects from ``v1`` serialize with an *apiVersion* of ``u"v1"``.
        """
        model = v1_5_model

        obj = model.v1.ComponentStatus()
        raw = model.iobject_to_raw(obj)
        self.expectThat(
            raw[u"apiVersion"],
            Equals(u"v1"),
        )
        self.expectThat(
            model.iobject_from_raw(raw),
            IsInstance(model.v1.ComponentStatus),
        )

    def test_v1beta1_apiVersion(self):
        """
        Objects from ``v1beta1`` serialize with an *apiVersion* of
        ``u"extensions/v1beta1"``.
        """
        model = v1_5_model

        obj = model.v1beta1.CertificateSigningRequest()
        raw = model.iobject_to_raw(obj)
        self.expectThat(
            raw[u"apiVersion"],
            Equals(u"extensions/v1beta1"),
        )
        self.expectThat(
            model.iobject_from_raw(raw),
            IsInstance(model.v1beta1.CertificateSigningRequest),
        )

class IObjectTests(TestCase):
    """
    Tests for ``IObject``.
    """
    @given(obj=iobjects())
    def test_interface(self, obj):
        """
        The object provides ``IObject``.
        """
        verifyObject(IObject, obj)

    @given(models())
    def test_constant_attributes(self, model):
        """
        The ``apiVersion`` and ``kind`` attributes reflect the Kubernetes object
        apiVersion and kind fields.
        """
        p = model.v1.Pod()
        self.expectThat(p.apiVersion, Equals(u"v1"))
        self.expectThat(p.kind, Equals(u"Pod"))

        pl = model.v1.PodList()
        self.expectThat(pl.apiVersion, Equals(u"v1"))
        self.expectThat(pl.kind, Equals(u"PodList"))

        d = model.v1beta1.Deployment()
        self.expectThat(d.apiVersion, Equals(u"v1beta1"))
        self.expectThat(d.kind, Equals(u"Deployment"))

        dl = model.v1beta1.DeploymentList()
        self.expectThat(dl.apiVersion, Equals(u"v1beta1"))
        self.expectThat(dl.kind, Equals(u"DeploymentList"))

    @given(obj=iobjects())
    def test_serialization_roundtrip(self, obj):
        """
        An ``IObject`` provider can be round-trip through JSON using
        ``iobject_to_raw`` and ``iobject_from_raw``.
        """
        model = v1_5_model

        marshalled = model.iobject_to_raw(obj)

        # Every IObject has these marshalled fields - and when looking at the
        # marshalled form, they're necessary to figure out the
        # schema/definition for the data.  We can't say anything in general
        # about the *values* (because of things like "extensions/v1beta1") but
        # we can at least assert the keys are present.
        self.expectThat(marshalled, ContainsAll([u"kind", u"apiVersion"]))

        # We should be able to unmarshal the data back to the same model
        # object as we started with.
        reloaded = model.iobject_from_raw(marshalled)
        self.expectThat(obj, PClassEquals(reloaded))

        # And, to be extra sure (ruling out any weird Python object
        # semantic hijinx), that that reconstituted object should marshal
        # back to exactly the same simplified object graph.
        remarshalled = model.iobject_to_raw(reloaded)
        self.expectThat(marshalled, MappingEquals(remarshalled))

        # Also, the marshalled form must be JSON compatible.
        serialized = dumps(marshalled)
        deserialized = loads(serialized)
        self.expectThat(marshalled, MappingEquals(deserialized))

    @given(objectcollections())
    def test_empty_collection(self, collection):
        """
        The ``items`` of a collection can be made empty in a couple different
        ways.
        """
        self.expectThat(collection.set(items=None).items, Equals([]))
        self.expectThat(collection.set(items=[]).items, Equals([]))

    @given(collection=namespacelists(), choose=choices())
    def test_unique_contents(self, collection, choose):
        """
        A collection type cannot contain more than one object with a particular
        namespace / name pair.
        """
        assume(len(collection.items) > 0)
        item = choose(collection.items)
        self.expectThat(
            lambda: collection.add(item),
            raises_exception(InvariantException),
        )

    def test_unknown_version(self):
        """
        ``iobject_from_raw`` raises ``UnrecognizedVersion`` if it does not
        recognize the *apiVersion* in the given data.
        """
        model = v1_5_model

        obj = {
            u"apiVersion": u"invalid.example.txkube",
            u"kind": u"Service",
        }
        self.assertThat(
            lambda: model.iobject_from_raw(obj),
            raises(UnrecognizedVersion(obj[u"apiVersion"], obj)),
        )

    def test_unknown_kind(self):
        """
        ``iobject_from_raw`` raises ``UnrecognizedKind`` if it does not recognize
        the *kind* in the given data.
        """
        model = v1_5_model

        obj = {
            u"apiVersion": u"v1",
            u"kind": u"SomethingFictional",
        }
        self.assertThat(
            lambda: model.iobject_from_raw(obj),
            raises(UnrecognizedKind(u"v1", u"SomethingFictional", obj)),
        )

class NamespaceTests(TestCase):
    """
    Other tests for ``Namespace``.
    """
    @given(models())
    def test_default(self, model):
        """
        ``Namespace.default`` returns the *default* namespace.
        """
        self.assertThat(
            model.v1.Namespace.default(),
            MatchesStructure(
                metadata=MatchesStructure(
                    name=Equals(u"default"),
                ),
            ),
        )

    @given(models())
    def test_fill_defaults(self, model):
        """
        ``Namespace.fill_defaults`` returns a ``Namespace`` with *uid* metadata
        and an active *status*.
        """
        # If they are not set already, a uid is generated and put into the
        # metadata and the status is set to active.
        sparse = model.v1.Namespace(metadata=model.v1.ObjectMeta(name=u"foo"))
        filled = sparse.fill_defaults()
        self.expectThat(
            filled,
            MatchesStructure(
                metadata=MatchesStructure(
                    uid=Not(Is(None)),
                ),
                status=Equals(model.v1.NamespaceStatus.active()),
            ),
        )

class NamespaceListTests(TestCase):
    """
    Tests for ``NamespaceList``.
    """
    @settings(suppress_health_check=[HealthCheck.exception_in_generation])
    @given(collection=namespacelists(), choose=choices())
    def test_remove(self, collection, choose):
        """
        ``NamespaceList.remove`` creates a new ``NamespaceList`` which does not
        have the given item.
        """
        assume(len(collection.items) > 0)
        item = choose(collection.items)
        removed = collection.remove(item)
        self.assertThat(removed.items, Not(Contains(item)))

    @given(collection=namespacelists(), choose=choices())
    def test_item_by_name(self, collection, choose):
        """
        ``NamespaceList.item_by_name`` returns the ``Namespace`` with the matching
        name.
        """
        assume(len(collection.items) > 0)
        for item in collection.items:
            self.expectThat(collection.item_by_name(item.metadata.name), Is(item))

        item = choose(collection.items)
        collection = collection.remove(item)
        self.expectThat(
            lambda: collection.item_by_name(item.metadata.name),
            raises(KeyError(item.metadata.name)),
        )

    @given(collection=namespacelists(), choose=choices())
    def test_no_duplicates(self, collection, choose):
        assume(len(collection.items) > 0)
        self.expectThat(
            lambda: collection.add(choose(collection.items)),
            raises_exception(InvariantException),
        )

    @given(collection=namespacelists())
    def test_constant_attributes(self, collection):
        self.expectThat(collection.kind, Equals(u"NamespaceList"))
        self.expectThat(collection.apiVersion, Equals(u"v1"))

class SetIfNoneTests(TestCase):
    """
    Tests for ``set_if_none``.
    """
    def test_none(self):
        """
        If the value for transformation is ``None``, the result contains the new
        value instead.
        """
        structure = freeze({u"foo": None})
        transformed = structure.transform([u"foo"], set_if_none(u"bar"))
        self.assertThat(transformed[u"foo"], Equals(u"bar"))

    def test_not_none(self):
        """
        If the value for transformation is not ``None``, the result contains the
        original value.
        """
        structure = freeze({u"foo": u"baz"})
        transformed = structure.transform([u"foo"], set_if_none(u"bar"))
        self.assertThat(transformed[u"foo"], Equals(u"baz"))

@implementer(IResponse)
@attr.s
class MemoryResponse(object):
    version = attr.ib()
    code = attr.ib()
    phrase = attr.ib()
    headers = attr.ib()
    length = attr.ib()

    request = attr.ib()
    previousResponse = attr.ib()

    _body = attr.ib()

    def deliverBody(self, protocol):
        protocol.makeConnection(None)
        protocol.dataReceived(self._body)
        protocol.connectionLost(Failure(ResponseDone()))

class KubernetesErrorTests(TestCase):
    """
    Tests for ``KubernetesError``.
    """
    def test_from_response(self):
        """
        ``from_response`` returns the same value as ``from_model_and_response``
        when called with the v1.5 model.
        """
        def response():
            body = dumps(v1_5_model.iobject_to_raw(v1_5_model.v1.Status()))
            return MemoryResponse(
                version=(b"HTTP", 1, 1),
                code=200,
                phrase=b"OK",
                headers=Headers(),
                length=len(body),
                request=None,
                previousResponse=None,
                body=body,
            )

        ds = gatherResults([
            KubernetesError.from_response(response()),
            KubernetesError.from_model_and_response(v1_5_model, response()),
        ])
        self.assertThat(
            ds,
            succeeded(EqualElements()),
        )

class Extra15DataModelTests(TestCase):
    """
    Tests for handling of certain Kubernetes 1.5 Swagger specifications.
    """
    def test_status(self):
        """
        If the Kubernetes-reported Swagger specification is missing the Status
        definitions, ``openapi_to_data_model`` returns a data model that
        defines them anyway.
        """
        openapi = v1_5_model.spec.to_document()
        openapi[u"definitions"].clear()
        model = openapi_to_data_model(openapi)
        model.v1.Status
        model.v1.StatusCause
        model.v1.StatusDetails
        model.v1.ListMeta
<|endoftext|>


<commit_before># coding: utf8
from __future__ import unicode_literals

import spacy
from spacy.tokens import Span, Doc

class CustomPipe:
    name = "my_pipe"

    def __init__(self):
        Span.set_extension("my_ext", getter=self._get_my_ext)
        Doc.set_extension("my_ext", default=None)

    def __call__(self, doc):
        gathered_ext = []
        for sent in doc.sents:
            sent_ext = self._get_my_ext(sent)
            sent._.set("my_ext", sent_ext)
            gathered_ext.append(sent_ext)

        doc._.set("my_ext", "\n".join(gathered_ext))

        return doc

    @staticmethod
    def _get_my_ext(span):
        return str(span.end)

def test_issue4903():
    # ensures that this runs correctly and doesn't hang or crash on Windows / macOS
    nlp = spacy.load("en_core_web_sm")
    custom_component = CustomPipe()
    nlp.add_pipe(custom_component, after="parser")

    text = ["I like bananas.", "Do you like them?", "No, I prefer wasabi."]
    # works without 'n_process'
    for doc in nlp.pipe(text, n_process=2):
        print(doc)
<commit_msg>remove old comment<commit_after># coding: utf8
from __future__ import unicode_literals

import spacy
from spacy.tokens import Span, Doc

class CustomPipe:
    name = "my_pipe"

    def __init__(self):
        Span.set_extension("my_ext", getter=self._get_my_ext)
        Doc.set_extension("my_ext", default=None)

    def __call__(self, doc):
        gathered_ext = []
        for sent in doc.sents:
            sent_ext = self._get_my_ext(sent)
            sent._.set("my_ext", sent_ext)
            gathered_ext.append(sent_ext)

        doc._.set("my_ext", "\n".join(gathered_ext))

        return doc

    @staticmethod
    def _get_my_ext(span):
        return str(span.end)

def test_issue4903():
    # ensures that this runs correctly and doesn't hang or crash on Windows / macOS
    nlp = spacy.load("en_core_web_sm")
    custom_component = CustomPipe()
    nlp.add_pipe(custom_component, after="parser")

    text = ["I like bananas.", "Do you like them?", "No, I prefer wasabi."]
    for doc in nlp.pipe(text, n_process=2):
        print(doc)
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

"""
Generator of histology report

"""
import logging
logger = logging.getLogger(__name__)

class LarVessels:
    def __init__(self):
        pass

    def add_cylinder(self, nodeA, nodeB, radius):
        pass

    def show(self):
        pass
<commit_msg>print node<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

"""
Generator of histology report

"""
import logging
logger = logging.getLogger(__name__)

class LarVessels:
    def __init__(self):
        pass

    def add_cylinder(self, nodeA, nodeB, radius):
        print 'nodeA ', nodeA
        pass

    def show(self):
        pass
<|endoftext|>


<commit_before>import datetime

import pytest

from sepadd import SepaDD

from .utils import clean_ids
from .utils import validate_xml

@pytest.fixture
def sdd():
    return SepaDD({
        "name": "Miller & Son Ltd",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "batch": True,
        "creditor_id": "DE26ZZZ00000000000",
        "currency": "EUR"
    }, schema="pain.008.003.02")

SAMPLE_RESULT = b"""
<Document xmlns="urn:iso:std:iso:20022:tech:xsd:pain.008.003.02" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <CstmrDrctDbtInitn>
    <GrpHdr>
      <MsgId>20012017014921-ba2dab283fdd</MsgId>
      <CreDtTm>2017-01-20T13:49:21</CreDtTm>
      <NbOfTxs>2</NbOfTxs>
      <CtrlSum>60.12</CtrlSum>
      <InitgPty>
        <Nm>Miller & Son Ltd</Nm>
        <Id>
          <OrgId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
            </Othr>
          </OrgId>
        </Id>
      </InitgPty>
    </GrpHdr>
    <PmtInf>
      <PmtInfId>MillerSonLtd-ecd6a2f680ce</PmtInfId>
      <PmtMtd>DD</PmtMtd>
      <BtchBookg>true</BtchBookg>
      <NbOfTxs>1</NbOfTxs>
      <CtrlSum>10.12</CtrlSum>
      <PmtTpInf>
        <SvcLvl>
          <Cd>SEPA</Cd>
        </SvcLvl>
        <LclInstrm>
          <Cd>CORE</Cd>
        </LclInstrm>
        <SeqTp>FRST</SeqTp>
      </PmtTpInf>
      <ReqdColltnDt>2017-01-20</ReqdColltnDt>
      <Cdtr>
        <Nm>Miller & Son Ltd</Nm>
      </Cdtr>
      <CdtrAcct>
        <Id>
          <IBAN>NL50BANK1234567890</IBAN>
        </Id>
      </CdtrAcct>
      <CdtrAgt>
        <FinInstnId>
          <BIC>BANKNL2A</BIC>
        </FinInstnId>
      </CdtrAgt>
      <ChrgBr>SLEV</ChrgBr>
      <CdtrSchmeId>
        <Id>
          <PrvtId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
              <SchmeNm>
                <Prtry>SEPA</Prtry>
              </SchmeNm>
            </Othr>
          </PrvtId>
        </Id>
      </CdtrSchmeId>
      <DrctDbtTxInf>
        <PmtId>
          <EndToEndId>MillerSonLtd-4431989789fb</EndToEndId>
        </PmtId>
        <InstdAmt Ccy="EUR">10.12</InstdAmt>
        <DrctDbtTx>
          <MndtRltdInf>
            <MndtId>1234</MndtId>
            <DtOfSgntr>2017-01-20</DtOfSgntr>
          </MndtRltdInf>
        </DrctDbtTx>
        <DbtrAgt>
          <FinInstnId>
            <BIC>BANKNL2A</BIC>
          </FinInstnId>
        </DbtrAgt>
        <Dbtr>
          <Nm>Test & Co.</Nm>
        </Dbtr>
        <DbtrAcct>
          <Id>
            <IBAN>NL50BANK1234567890</IBAN>
          </Id>
        </DbtrAcct>
        <RmtInf>
          <Ustrd>Test transaction1</Ustrd>
        </RmtInf>
      </DrctDbtTxInf>
    </PmtInf>
    <PmtInf>
      <PmtInfId>MillerSonLtd-d547a1b3882f</PmtInfId>
      <PmtMtd>DD</PmtMtd>
      <BtchBookg>true</BtchBookg>
      <NbOfTxs>1</NbOfTxs>
      <CtrlSum>50.00</CtrlSum>
      <PmtTpInf>
        <SvcLvl>
          <Cd>SEPA</Cd>
        </SvcLvl>
        <LclInstrm>
          <Cd>CORE</Cd>
        </LclInstrm>
        <SeqTp>RCUR</SeqTp>
      </PmtTpInf>
      <ReqdColltnDt>2017-01-20</ReqdColltnDt>
      <Cdtr>
        <Nm>Miller & Son Ltd</Nm>
      </Cdtr>
      <CdtrAcct>
        <Id>
          <IBAN>NL50BANK1234567890</IBAN>
        </Id>
      </CdtrAcct>
      <CdtrAgt>
        <FinInstnId>
          <BIC>BANKNL2A</BIC>
        </FinInstnId>
      </CdtrAgt>
      <ChrgBr>SLEV</ChrgBr>
      <CdtrSchmeId>
        <Id>
          <PrvtId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
              <SchmeNm>
                <Prtry>SEPA</Prtry>
              </SchmeNm>
            </Othr>
          </PrvtId>
        </Id>
      </CdtrSchmeId>
      <DrctDbtTxInf>
        <PmtId>
          <EndToEndId>MillerSonLtd-7e989083e265</EndToEndId>
        </PmtId>
        <InstdAmt Ccy="EUR">50.00</InstdAmt>
        <DrctDbtTx>
          <MndtRltdInf>
            <MndtId>1234</MndtId>
            <DtOfSgntr>2017-01-20</DtOfSgntr>
          </MndtRltdInf>
        </DrctDbtTx>
        <DbtrAgt>
          <FinInstnId>
            <BIC>BANKNL2A</BIC>
          </FinInstnId>
        </DbtrAgt>
        <Dbtr>
          <Nm>Test du Test</Nm>
        </Dbtr>
        <DbtrAcct>
          <Id>
            <IBAN>NL50BANK1234567890</IBAN>
          </Id>
        </DbtrAcct>
        <RmtInf>
          <Ustrd>Testgrüße <html></Ustrd>
        </RmtInf>
      </DrctDbtTxInf>
    </PmtInf>
  </CstmrDrctDbtInitn>
</Document>
"""

def test_two_debits(sdd):
    payment1 = {
        "name": "Test & Co.",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "amount": 1012,
        "type": "FRST",
        "collection_date": datetime.date.today(),
        "mandate_id": "1234",
        "mandate_date": datetime.date.today(),
        "description": "Test transaction1"
    }
    payment2 = {
        "name": "Test du Test",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "amount": 5000,
        "type": "RCUR",
        "collection_date": datetime.date.today(),
        "mandate_id": "1234",
        "mandate_date": datetime.date.today(),
        "description": "Testgrüße <html>"
    }

    sdd.add_payment(payment1)
    sdd.add_payment(payment2)
    xmlout = sdd.export()
    xmlpretty = validate_xml(xmlout, "pain.008.003.02")
    assert clean_ids(xmlpretty.strip()) == clean_ids(SAMPLE_RESULT.strip())
<commit_msg>Fix import order<commit_after>import datetime

import pytest

from sepadd import SepaDD

from .utils import clean_ids, validate_xml

@pytest.fixture
def sdd():
    return SepaDD({
        "name": "Miller & Son Ltd",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "batch": True,
        "creditor_id": "DE26ZZZ00000000000",
        "currency": "EUR"
    }, schema="pain.008.003.02")

SAMPLE_RESULT = b"""
<Document xmlns="urn:iso:std:iso:20022:tech:xsd:pain.008.003.02" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
  <CstmrDrctDbtInitn>
    <GrpHdr>
      <MsgId>20012017014921-ba2dab283fdd</MsgId>
      <CreDtTm>2017-01-20T13:49:21</CreDtTm>
      <NbOfTxs>2</NbOfTxs>
      <CtrlSum>60.12</CtrlSum>
      <InitgPty>
        <Nm>Miller & Son Ltd</Nm>
        <Id>
          <OrgId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
            </Othr>
          </OrgId>
        </Id>
      </InitgPty>
    </GrpHdr>
    <PmtInf>
      <PmtInfId>MillerSonLtd-ecd6a2f680ce</PmtInfId>
      <PmtMtd>DD</PmtMtd>
      <BtchBookg>true</BtchBookg>
      <NbOfTxs>1</NbOfTxs>
      <CtrlSum>10.12</CtrlSum>
      <PmtTpInf>
        <SvcLvl>
          <Cd>SEPA</Cd>
        </SvcLvl>
        <LclInstrm>
          <Cd>CORE</Cd>
        </LclInstrm>
        <SeqTp>FRST</SeqTp>
      </PmtTpInf>
      <ReqdColltnDt>2017-01-20</ReqdColltnDt>
      <Cdtr>
        <Nm>Miller & Son Ltd</Nm>
      </Cdtr>
      <CdtrAcct>
        <Id>
          <IBAN>NL50BANK1234567890</IBAN>
        </Id>
      </CdtrAcct>
      <CdtrAgt>
        <FinInstnId>
          <BIC>BANKNL2A</BIC>
        </FinInstnId>
      </CdtrAgt>
      <ChrgBr>SLEV</ChrgBr>
      <CdtrSchmeId>
        <Id>
          <PrvtId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
              <SchmeNm>
                <Prtry>SEPA</Prtry>
              </SchmeNm>
            </Othr>
          </PrvtId>
        </Id>
      </CdtrSchmeId>
      <DrctDbtTxInf>
        <PmtId>
          <EndToEndId>MillerSonLtd-4431989789fb</EndToEndId>
        </PmtId>
        <InstdAmt Ccy="EUR">10.12</InstdAmt>
        <DrctDbtTx>
          <MndtRltdInf>
            <MndtId>1234</MndtId>
            <DtOfSgntr>2017-01-20</DtOfSgntr>
          </MndtRltdInf>
        </DrctDbtTx>
        <DbtrAgt>
          <FinInstnId>
            <BIC>BANKNL2A</BIC>
          </FinInstnId>
        </DbtrAgt>
        <Dbtr>
          <Nm>Test & Co.</Nm>
        </Dbtr>
        <DbtrAcct>
          <Id>
            <IBAN>NL50BANK1234567890</IBAN>
          </Id>
        </DbtrAcct>
        <RmtInf>
          <Ustrd>Test transaction1</Ustrd>
        </RmtInf>
      </DrctDbtTxInf>
    </PmtInf>
    <PmtInf>
      <PmtInfId>MillerSonLtd-d547a1b3882f</PmtInfId>
      <PmtMtd>DD</PmtMtd>
      <BtchBookg>true</BtchBookg>
      <NbOfTxs>1</NbOfTxs>
      <CtrlSum>50.00</CtrlSum>
      <PmtTpInf>
        <SvcLvl>
          <Cd>SEPA</Cd>
        </SvcLvl>
        <LclInstrm>
          <Cd>CORE</Cd>
        </LclInstrm>
        <SeqTp>RCUR</SeqTp>
      </PmtTpInf>
      <ReqdColltnDt>2017-01-20</ReqdColltnDt>
      <Cdtr>
        <Nm>Miller & Son Ltd</Nm>
      </Cdtr>
      <CdtrAcct>
        <Id>
          <IBAN>NL50BANK1234567890</IBAN>
        </Id>
      </CdtrAcct>
      <CdtrAgt>
        <FinInstnId>
          <BIC>BANKNL2A</BIC>
        </FinInstnId>
      </CdtrAgt>
      <ChrgBr>SLEV</ChrgBr>
      <CdtrSchmeId>
        <Id>
          <PrvtId>
            <Othr>
              <Id>DE26ZZZ00000000000</Id>
              <SchmeNm>
                <Prtry>SEPA</Prtry>
              </SchmeNm>
            </Othr>
          </PrvtId>
        </Id>
      </CdtrSchmeId>
      <DrctDbtTxInf>
        <PmtId>
          <EndToEndId>MillerSonLtd-7e989083e265</EndToEndId>
        </PmtId>
        <InstdAmt Ccy="EUR">50.00</InstdAmt>
        <DrctDbtTx>
          <MndtRltdInf>
            <MndtId>1234</MndtId>
            <DtOfSgntr>2017-01-20</DtOfSgntr>
          </MndtRltdInf>
        </DrctDbtTx>
        <DbtrAgt>
          <FinInstnId>
            <BIC>BANKNL2A</BIC>
          </FinInstnId>
        </DbtrAgt>
        <Dbtr>
          <Nm>Test du Test</Nm>
        </Dbtr>
        <DbtrAcct>
          <Id>
            <IBAN>NL50BANK1234567890</IBAN>
          </Id>
        </DbtrAcct>
        <RmtInf>
          <Ustrd>Testgrüße <html></Ustrd>
        </RmtInf>
      </DrctDbtTxInf>
    </PmtInf>
  </CstmrDrctDbtInitn>
</Document>
"""

def test_two_debits(sdd):
    payment1 = {
        "name": "Test & Co.",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "amount": 1012,
        "type": "FRST",
        "collection_date": datetime.date.today(),
        "mandate_id": "1234",
        "mandate_date": datetime.date.today(),
        "description": "Test transaction1"
    }
    payment2 = {
        "name": "Test du Test",
        "IBAN": "NL50BANK1234567890",
        "BIC": "BANKNL2A",
        "amount": 5000,
        "type": "RCUR",
        "collection_date": datetime.date.today(),
        "mandate_id": "1234",
        "mandate_date": datetime.date.today(),
        "description": "Testgrüße <html>"
    }

    sdd.add_payment(payment1)
    sdd.add_payment(payment2)
    xmlout = sdd.export()
    xmlpretty = validate_xml(xmlout, "pain.008.003.02")
    assert clean_ids(xmlpretty.strip()) == clean_ids(SAMPLE_RESULT.strip())
<|endoftext|>