<commit_before># -*- encoding: utf-8 -*-
#
# Copyright © 2013 eNovance <licensing@enovance.com>
#
# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import uuid

from oslo.config import cfg

from ceilometer.alarm import rpc as rpc_alarm
from ceilometer.openstack.common import rpc
from ceilometer.storage.models import Alarm as AlarmModel
from ceilometer.tests import base
from ceilometerclient.v2.alarms import Alarm as AlarmClient

class TestRPCAlarmNotifier(base.TestCase):
    def faux_cast(self, context, topic, msg):
        self.notified.append((topic, msg))

    def setUp(self):
        super(TestRPCAlarmNotifier, self).setUp()
        self.notified = []
        self.stubs.Set(rpc, 'cast', self.faux_cast)
        self.notifier = rpc_alarm.RPCAlarmNotifier()
        self.alarms = [
            AlarmClient(None, info={
                'name': 'instance_running_hot',
                'meter_name': 'cpu_util',
                'comparison_operator': 'gt',
                'threshold': 80.0,
                'evaluation_periods': 5,
                'statistic': 'avg',
                'state': 'ok',
                'ok_actions': ['http://host:8080/path'],
                'user_id': 'foobar',
                'project_id': 'snafu',
                'period': 60,
                'alarm_id': str(uuid.uuid4()),
                'matching_metadata':{'resource_id':
                                     'my_instance'}
            }),
            AlarmClient(None, info={
                'name': 'group_running_idle',
                'meter_name': 'cpu_util',
                'comparison_operator': 'le',
                'threshold': 10.0,
                'statistic': 'max',
                'evaluation_periods': 4,
                'state': 'insufficient data',
                'insufficient_data_actions': ['http://other_host/path'],
                'user_id': 'foobar',
                'project_id': 'snafu',
                'period': 300,
                'alarm_id': str(uuid.uuid4()),
                'matching_metadata':{'metadata.user_metadata.AS':
                                     'my_group'}
            }),
        ]

    def test_notify_alarm(self):
        previous = ['alarm', 'ok']
        for i, a in enumerate(self.alarms):
            self.notifier.notify(a, previous[i], "what? %d" % i)
        self.assertEqual(len(self.notified), 2)
        for i, a in enumerate(self.alarms):
            actions = getattr(a, AlarmModel.ALARM_ACTIONS_MAP[a.state])
            self.assertEqual(self.notified[i][0],
                             cfg.CONF.alarm.notifier_rpc_topic)
            self.assertEqual(self.notified[i][1]["args"]["data"]["alarm_id"],
                             self.alarms[i].alarm_id)
            self.assertEqual(self.notified[i][1]["args"]["data"]["actions"],
                             actions)
            self.assertEqual(self.notified[i][1]["args"]["data"]["previous"],
                             previous[i])
            self.assertEqual(self.notified[i][1]["args"]["data"]["current"],
                             self.alarms[i].state)
            self.assertEqual(self.notified[i][1]["args"]["data"]["reason"],
                             "what? %d" % i)

    def test_notify_non_string_reason(self):
        self.notifier.notify(self.alarms[0], 'ok', 42)
        reason = self.notified[0][1]['args']['data']['reason']
        self.assertTrue(isinstance(reason, basestring))

    def test_notify_no_actions(self):
        alarm = AlarmClient(None, info={
            'name': 'instance_running_hot',
            'meter_name': 'cpu_util',
            'comparison_operator': 'gt',
            'threshold': 80.0,
            'evaluation_periods': 5,
            'statistic': 'avg',
            'state': 'ok',
            'user_id': 'foobar',
            'project_id': 'snafu',
            'period': 60,
            'ok_actions': [],
            'alarm_id': str(uuid.uuid4()),
            'matching_metadata': {'resource_id':
                                  'my_instance'}
        })
        self.notifier.notify(alarm, 'alarm', "what?")
        self.assertEqual(len(self.notified), 0)
<commit_msg>add more test cases to improve the test code coverage #6<commit_after># -*- encoding: utf-8 -*-
#
# Copyright © 2013 eNovance <licensing@enovance.com>
#
# Authors: Mehdi Abaakouk <mehdi.abaakouk@enovance.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

import uuid
import mock

from oslo.config import cfg

from ceilometer.alarm import rpc as rpc_alarm
from ceilometer.openstack.common import rpc
from ceilometer.openstack.common import timeutils
from ceilometer.storage.models import Alarm as AlarmModel
from ceilometer.tests import base
from ceilometerclient.v2.alarms import Alarm as AlarmClient

class TestRPCAlarmNotifier(base.TestCase):
    def faux_cast(self, context, topic, msg):
        self.notified.append((topic, msg))

    def setUp(self):
        super(TestRPCAlarmNotifier, self).setUp()
        self.notified = []
        self.stubs.Set(rpc, 'cast', self.faux_cast)
        self.notifier = rpc_alarm.RPCAlarmNotifier()
        self.alarms = [
            AlarmClient(None, info={
                'name': 'instance_running_hot',
                'meter_name': 'cpu_util',
                'comparison_operator': 'gt',
                'threshold': 80.0,
                'evaluation_periods': 5,
                'statistic': 'avg',
                'state': 'ok',
                'ok_actions': ['http://host:8080/path'],
                'user_id': 'foobar',
                'project_id': 'snafu',
                'period': 60,
                'alarm_id': str(uuid.uuid4()),
                'matching_metadata':{'resource_id':
                                     'my_instance'}
            }),
            AlarmClient(None, info={
                'name': 'group_running_idle',
                'meter_name': 'cpu_util',
                'comparison_operator': 'le',
                'threshold': 10.0,
                'statistic': 'max',
                'evaluation_periods': 4,
                'state': 'insufficient data',
                'insufficient_data_actions': ['http://other_host/path'],
                'user_id': 'foobar',
                'project_id': 'snafu',
                'period': 300,
                'alarm_id': str(uuid.uuid4()),
                'matching_metadata':{'metadata.user_metadata.AS':
                                     'my_group'}
            }),
        ]

    def test_notify_alarm(self):
        previous = ['alarm', 'ok']
        for i, a in enumerate(self.alarms):
            self.notifier.notify(a, previous[i], "what? %d" % i)
        self.assertEqual(len(self.notified), 2)
        for i, a in enumerate(self.alarms):
            actions = getattr(a, AlarmModel.ALARM_ACTIONS_MAP[a.state])
            self.assertEqual(self.notified[i][0],
                             cfg.CONF.alarm.notifier_rpc_topic)
            self.assertEqual(self.notified[i][1]["args"]["data"]["alarm_id"],
                             self.alarms[i].alarm_id)
            self.assertEqual(self.notified[i][1]["args"]["data"]["actions"],
                             actions)
            self.assertEqual(self.notified[i][1]["args"]["data"]["previous"],
                             previous[i])
            self.assertEqual(self.notified[i][1]["args"]["data"]["current"],
                             self.alarms[i].state)
            self.assertEqual(self.notified[i][1]["args"]["data"]["reason"],
                             "what? %d" % i)

    def test_notify_non_string_reason(self):
        self.notifier.notify(self.alarms[0], 'ok', 42)
        reason = self.notified[0][1]['args']['data']['reason']
        self.assertTrue(isinstance(reason, basestring))

    def test_notify_no_actions(self):
        alarm = AlarmClient(None, info={
            'name': 'instance_running_hot',
            'meter_name': 'cpu_util',
            'comparison_operator': 'gt',
            'threshold': 80.0,
            'evaluation_periods': 5,
            'statistic': 'avg',
            'state': 'ok',
            'user_id': 'foobar',
            'project_id': 'snafu',
            'period': 60,
            'ok_actions': [],
            'alarm_id': str(uuid.uuid4()),
            'matching_metadata': {'resource_id':
                                  'my_instance'}
        })
        self.notifier.notify(alarm, 'alarm', "what?")
        self.assertEqual(len(self.notified), 0)

class TestRPCAlarmPartitionCoordination(base.TestCase):
    def faux_fanout_cast(self, context, topic, msg):
        self.notified.append((topic, msg))

    def setUp(self):
        super(TestRPCAlarmPartitionCoordination, self).setUp()
        self.notified = []
        self.stubs.Set(rpc, 'fanout_cast', self.faux_fanout_cast)
        self.ordination = rpc_alarm.RPCAlarmPartitionCoordination()
        self.alarms = [mock.MagicMock(), mock.MagicMock()]

    def test_ordination_presence(self):
        id = uuid.uuid4()
        priority = float(timeutils.utcnow().strftime('%s.%f'))
        self.ordination.presence(id, priority)
        topic, msg = self.notified[0]
        self.assertEqual(topic, 'alarm_partition_coordination')
        self.assertEqual(msg['args']['data']['uuid'], id)
        self.assertEqual(msg['args']['data']['priority'], priority)
        self.assertEqual(msg['method'], 'presence')

    def test_ordination_assign(self):
        id = uuid.uuid4()
        self.ordination.assign(id, self.alarms)
        topic, msg = self.notified[0]
        self.assertEqual(topic, 'alarm_partition_coordination')
        self.assertEqual(msg['args']['data']['uuid'], id)
        self.assertEqual(len(msg['args']['data']['alarms']), 2)
        self.assertEqual(msg['method'], 'assign')

    def test_ordination_allocate(self):
        id = uuid.uuid4()
        self.ordination.allocate(id, self.alarms)
        topic, msg = self.notified[0]
        self.assertEqual(topic, 'alarm_partition_coordination')
        self.assertEqual(msg['args']['data']['uuid'], id)
        self.assertEqual(len(msg['args']['data']['alarms']), 2)
        self.assertEqual(msg['method'], 'allocate')
<|endoftext|>


<commit_before>s = {1}
print(s.pop())
try:
    print(s.pop(), "!!!")
except KeyError:
    pass
else:
    print("Failed to raise KeyError")

# this tests an optimisation in mp_set_remove_first
# N must not be equal to one of the values in hash_allocation_sizes
N = 11
s = set(range(N))
while s:
    print(s.pop()) # last pop() should trigger the optimisation
for i in range(N):
    s.add(i) # check that we can add the numbers back to the set
print(list(s))
<commit_msg>tests/basics/set_pop.py: Sort set before printing for consistent output.<commit_after>s = {1}
print(s.pop())
try:
    print(s.pop(), "!!!")
except KeyError:
    pass
else:
    print("Failed to raise KeyError")

# this tests an optimisation in mp_set_remove_first
# N must not be equal to one of the values in hash_allocation_sizes
N = 11
s = set(range(N))
while s:
    print(s.pop()) # last pop() should trigger the optimisation
for i in range(N):
    s.add(i) # check that we can add the numbers back to the set
print(sorted(s))
<|endoftext|>


<commit_before>import json
import os
import shutil
import tempfile

import pytest

import rasa.data as data
from tests.core.conftest import DEFAULT_STORIES_FILE, DEFAULT_NLU_DATA

def test_get_core_directory(project):
    data_dir = os.path.join(project, "data")
    core_directory = data.get_core_directory([data_dir])
    stories = os.listdir(core_directory)

    assert len(stories) == 1
    assert stories[0].endswith("stories.md")

def test_get_nlu_directory(project):
    data_dir = os.path.join(project, "data")
    nlu_directory = data.get_nlu_directory([data_dir])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 1
    assert nlu_files[0].endswith("nlu.md")

def test_get_nlu_file(project):
    data_file = os.path.join(project, "data/nlu.md")
    nlu_directory = data.get_nlu_directory(data_file)

    assert os.listdir(nlu_directory) == ["nlu.md"]

def test_get_core_nlu_directories(project):
    data_dir = os.path.join(project, "data")
    core_directory, nlu_directory = data.get_core_nlu_directories([data_dir])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 1
    assert nlu_files[0].endswith("nlu.md")

    stories = os.listdir(core_directory)

    assert len(stories) == 1
    assert stories[0].endswith("stories.md")

def test_same_file_names_get_resolved(tmpdir):
    # makes sure the resolution properly handles if there are two files with
    # with the same name in different directories

    tmpdir.join("one").mkdir()
    tmpdir.join("two").mkdir()
    data_dir_one = os.path.join(tmpdir.join("one").join("stories.md").strpath)
    data_dir_two = os.path.join(tmpdir.join("two").join("stories.md").strpath)
    shutil.copy2(DEFAULT_STORIES_FILE, data_dir_one)
    shutil.copy2(DEFAULT_STORIES_FILE, data_dir_two)

    nlu_dir_one = os.path.join(tmpdir.join("one").join("nlu.md").strpath)
    nlu_dir_two = os.path.join(tmpdir.join("two").join("nlu.md").strpath)
    shutil.copy2(DEFAULT_NLU_DATA, nlu_dir_one)
    shutil.copy2(DEFAULT_NLU_DATA, nlu_dir_two)

    core_directory, nlu_directory = data.get_core_nlu_directories([tmpdir.strpath])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 2
    assert all([f.endswith("nlu.md") for f in nlu_files])

    stories = os.listdir(core_directory)

    assert len(stories) == 2
    assert all([f.endswith("stories.md") for f in stories])

@pytest.mark.parametrize(
    "line",
    [
        "##intent:aintent",
        "##synonym: synonym",
        "##regex:a_regex",
        " ##lookup:additional",
    ],
)
def test_contains_nlu_pattern(line):
    assert data._contains_nlu_pattern(line)

def test_is_nlu_file_with_json():
    test = {
        "rasa_nlu_data": {
            "lookup_tables": [
                {"name": "plates", "elements": ["beans", "rice", "tacos", "cheese"]}
            ]
        }
    }

    directory = tempfile.mkdtemp()
    file = os.path.join(directory, "test.json")
    with open(file, "w") as f:
        f.write(json.dumps(test))

    assert data._is_nlu_file(file)

def test_is_not_nlu_file_with_json():
    directory = tempfile.mkdtemp()
    file = os.path.join(directory, "test.json")
    with open(file, "w") as f:
        f.write('{"test": "a"}')

    assert not data._is_nlu_file(file)

@pytest.mark.parametrize("line", ["- example", "## story intent 1 + two##slots* entry"])
def test_not_contains_nlu_pattern(line):
    assert not data._contains_nlu_pattern(line)
<commit_msg>black reformat, make test more thorough<commit_after>import json
import os
import shutil
import tempfile

import pytest

import rasa.data as data
from tests.core.conftest import DEFAULT_STORIES_FILE, DEFAULT_NLU_DATA
from rasa.nlu.training_data import load_data

def test_get_core_directory(project):
    data_dir = os.path.join(project, "data")
    core_directory = data.get_core_directory([data_dir])
    stories = os.listdir(core_directory)

    assert len(stories) == 1
    assert stories[0].endswith("stories.md")

def test_get_nlu_directory(project):
    data_dir = os.path.join(project, "data")
    nlu_directory = data.get_nlu_directory([data_dir])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 1
    assert nlu_files[0].endswith("nlu.md")

def test_get_nlu_file(project):
    data_file = os.path.join(project, "data/nlu.md")
    nlu_directory = data.get_nlu_directory(data_file)

    nlu_files = os.listdir(nlu_directory)

    original = load_data(data_file)
    copied = load_data(nlu_directory)

    assert nlu_files[0].endswith("nlu.md")
    assert original.intent_examples == copied.intent_examples

def test_get_core_nlu_directories(project):
    data_dir = os.path.join(project, "data")
    core_directory, nlu_directory = data.get_core_nlu_directories([data_dir])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 1
    assert nlu_files[0].endswith("nlu.md")

    stories = os.listdir(core_directory)

    assert len(stories) == 1
    assert stories[0].endswith("stories.md")

def test_same_file_names_get_resolved(tmpdir):
    # makes sure the resolution properly handles if there are two files with
    # with the same name in different directories

    tmpdir.join("one").mkdir()
    tmpdir.join("two").mkdir()
    data_dir_one = os.path.join(tmpdir.join("one").join("stories.md").strpath)
    data_dir_two = os.path.join(tmpdir.join("two").join("stories.md").strpath)
    shutil.copy2(DEFAULT_STORIES_FILE, data_dir_one)
    shutil.copy2(DEFAULT_STORIES_FILE, data_dir_two)

    nlu_dir_one = os.path.join(tmpdir.join("one").join("nlu.md").strpath)
    nlu_dir_two = os.path.join(tmpdir.join("two").join("nlu.md").strpath)
    shutil.copy2(DEFAULT_NLU_DATA, nlu_dir_one)
    shutil.copy2(DEFAULT_NLU_DATA, nlu_dir_two)

    core_directory, nlu_directory = data.get_core_nlu_directories([tmpdir.strpath])

    nlu_files = os.listdir(nlu_directory)

    assert len(nlu_files) == 2
    assert all([f.endswith("nlu.md") for f in nlu_files])

    stories = os.listdir(core_directory)

    assert len(stories) == 2
    assert all([f.endswith("stories.md") for f in stories])

@pytest.mark.parametrize(
    "line",
    [
        "##intent:aintent",
        "##synonym: synonym",
        "##regex:a_regex",
        " ##lookup:additional",
    ],
)
def test_contains_nlu_pattern(line):
    assert data._contains_nlu_pattern(line)

def test_is_nlu_file_with_json():
    test = {
        "rasa_nlu_data": {
            "lookup_tables": [
                {"name": "plates", "elements": ["beans", "rice", "tacos", "cheese"]}
            ]
        }
    }

    directory = tempfile.mkdtemp()
    file = os.path.join(directory, "test.json")
    with open(file, "w") as f:
        f.write(json.dumps(test))

    assert data._is_nlu_file(file)

def test_is_not_nlu_file_with_json():
    directory = tempfile.mkdtemp()
    file = os.path.join(directory, "test.json")
    with open(file, "w") as f:
        f.write('{"test": "a"}')

    assert not data._is_nlu_file(file)

@pytest.mark.parametrize("line", ["- example", "## story intent 1 + two##slots* entry"])
def test_not_contains_nlu_pattern(line):
    assert not data._contains_nlu_pattern(line)
<|endoftext|>


<commit_before>#!/usr/bin/env python2.7

import malibu, sqlite3, unittest
from malibu.database import dbmapper
from nose.tools import *

class DBMapperTestCase(unittest.TestCase):

    def setUp(self):

        self.db = sqlite3.connect(":memory:")

    def tearDown(self):

        self.db.close()

    def recordCreate_test(self):

        dbm = DBMap(self.db)
        dbm.set_test_col("Test")
        dbm.set_example(True)
        dbm.set_description("This is a test.")

        self.assertEquals(dbm._test_col, "Test")
        self.assertEquals(dbm._example, True)
        self.assertEquals(dbm._description, "This is a test.")

    def recordLoad_test(self):

        dbm = DBMap(self.db)

        dbm.create()
        load_id = dbm.get_id()

        dbm.set_test_col("Test")
        dbm.set_example(True)
        dbm.set_description("This is a test.")
        dbm.save()

        del dbm

        dbm = DBMap(self.db)
        record = DBMap.load(id = load_id)

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, True)
        self.assertEquals(record._description, "This is a test.")

    def recordNew_test(self):

        dbm = DBMap(self.db)

        record = DBMap.new(test_col = "Test", example = False, description = "This is not a test.")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

    def recordNewLoad_test(self):

        dbm = DBMap(self.db)

        record = DBMap.new(test_col = "Test", example = False, description = "This is not a test.")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

        record.save()
        del record

        record = DBMap.load(test_col = "Test")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

    def recordFind_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find(example = False)

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

    def recordFindAll_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

    def recordFilterEquals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_equals("test_col", "TestA")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterIequals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_iequals("test_col", "testa")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterInequals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_inequals("test_col", "TestB")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterRegex_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_regex("test_col", "Test[AB]")

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

class DBMap(dbmapper.DBMapper):

    def __init__(self, db):

        keys = ['id', 'test_col', 'example', 'description']
        ktypes = ['integer', 'text', 'boolean', 'text']

        DBMap.set_db_options(db, keys, ktypes)

        dbmapper.DBMapper.__init__(self, db, keys, ktypes)
<commit_msg>1.0: updated dbmapper tests.<commit_after>#!/usr/bin/env python2.7

import malibu, sqlite3, unittest
from malibu.database import dbmapper
from nose.tools import *

class DBMapperTestCase(unittest.TestCase):

    def setUp(self):

        self.db = sqlite3.connect(":memory:")

    def tearDown(self):

        self.db.close()

    def recordCreate_test(self):

        dbm = DBMap(self.db)
        dbm.set_test_col("Test")
        dbm.set_example(True)
        dbm.set_description("This is a test.")

        self.assertEquals(dbm._test_col, "Test")
        self.assertEquals(dbm._example, True)
        self.assertEquals(dbm._description, "This is a test.")

    def recordLoad_test(self):

        dbm = DBMap(self.db)

        dbm.create()
        load_id = dbm.get_id()

        dbm.set_test_col("Test")
        dbm.set_example(True)
        dbm.set_description("This is a test.")

        del dbm

        dbm = DBMap(self.db)
        record = DBMap.load(id = load_id)

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, True)
        self.assertEquals(record._description, "This is a test.")

    def recordNew_test(self):

        dbm = DBMap(self.db)

        record = DBMap.new(test_col = "Test", example = False, description = "This is not a test.")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

    def recordNewLoad_test(self):

        dbm = DBMap(self.db)

        record = DBMap.new(test_col = "Test", example = False, description = "This is not a test.")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

        del record

        record = DBMap.load(test_col = "Test")

        self.assertEquals(record._test_col, "Test")
        self.assertEquals(record._example, False)
        self.assertEquals(record._description, "This is not a test.")

    def recordFind_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find(example = False)

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

    def recordFindAll_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

    def recordFilterEquals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_equals("test_col", "TestA")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterIequals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_iequals("test_col", "testa")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterInequals_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_inequals("test_col", "TestB")

        self.assertEquals(len(result), 1)
        self.assertEquals(result[0]._test_col, "TestA")

    def recordFilterRegex_test(self):

        dbm = DBMap(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        result = DBMap.find_all()
        result = result.filter_regex("test_col", "Test[AB]")

        self.assertEquals(len(result), 2)
        self.assertEquals(result[0]._test_col, "TestA")
        self.assertEquals(result[1]._test_col, "TestB")

    def recordJoin_test(self):

        dbm = DBMap(self.db)
        dbl = DBMapLink(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        id_a = DBMap.load(test_col = "TestA").get_id()
        id_b = DBMap.load(test_col = "TestB").get_id()

        DBMapLink.new(map_id = id_a, some_text = "This definitely is not a test.")
        DBMapLink.new(map_id = id_b, some_text = "This might be a test.")
        DBMapLink.new(map_id = id_a, some_text = "This could be a test.")

        result = DBMap.join(DBMapLink, "id", "map_id")

        self.assertEquals(len(result), 3)
        self.assertEquals(result[0][0]._id, 1)
        self.assertEquals(result[0][1]._id, 1)
        self.assertEquals(result[1][0]._id, 2)
        self.assertEquals(result[1][1]._id, 2)
        self.assertEquals(result[2][0]._id, 1)
        self.assertEquals(result[2][1]._id, 3)

    def recordJoinFilter_test(self):

        dbm = DBMap(self.db)
        dbl = DBMapLink(self.db)

        DBMap.new(test_col = "TestA", example = False, description = "This is not a test.")
        DBMap.new(test_col = "TestB", example = False, description = "This is still not a test.")

        id_a = DBMap.load(test_col = "TestA").get_id()
        id_b = DBMap.load(test_col = "TestB").get_id()

        DBMapLink.new(map_id = id_a, some_text = "This definitely is not a test.")
        DBMapLink.new(map_id = id_b, some_text = "This might be a test.")
        DBMapLink.new(map_id = id_a, some_text = "This could be a test.")

        result = DBMap.join(DBMapLink, "id", "map_id")

        self.assertEquals(len(result), 3)

class DBMap(dbmapper.DBMapper):

    def __init__(self, db):

        keys = ['id', 'test_col', 'example', 'description']
        ktypes = ['integer', 'text', 'boolean', 'text']

        DBMap.set_db_options(db, keys, ktypes)

        dbmapper.DBMapper.__init__(self, db, keys, ktypes)

class DBMapLink(dbmapper.DBMapper):

    def __init__(self, db):

        keys = ['id', 'map_id', 'some_text']
        ktypes = ['integer', 'integer', 'text']

        DBMapLink.set_db_options(db, keys, ktypes)

        dbmapper.DBMapper.__init__(self, db, keys, ktypes)
<|endoftext|>


<commit_before>import boto
import json
import os
import pytest

from boto.s3.key import Key as S3Key
from boto.exception import NoAuthHandlerFound
from os.path import split, splitext, abspath, isfile, join, relpath

from ..utils import warn, fail, write, green
from ..constants import __version__, s3, s3_bucket, example_dir, build_id

from .collect_examples import get_all_examples

def no_ext(path):
    return splitext(abspath(path))[0]

def get_example_pngs(example_file):
    diff = pytest.config.option.diff
    example_path = no_ext(example_file)
    test_png = "%s-%s-%s.png" % (example_path, __version__, build_id)
    if diff:
        ref_png = "%s-%s-%s.png" % (example_path, diff, build_id)
        diff_png = "%s-%s-%s-diff-%s.png" % (example_path, __version__, diff, build_id)
    else:
        ref_png = None
        diff_png = None
    return (test_png, ref_png, diff_png)

def _upload_image(bucket, path, s3_png_file):
    with open(path, 'rb') as f:
        png = f.read()
    write("%s Uploading image to S3 to %s/%s" % (green(">>>"), s3, s3_png_file))
    key = S3Key(bucket, s3_png_file)
    key.set_metadata("Content-Type", "image/png")
    key.set_contents_from_string(png, policy="public-read")

def upload_example_pngs_to_s3():
    # Test connection
    try:
        conn = boto.connect_s3()
        bucket = conn.get_bucket(s3_bucket)
        upload = True
    except NoAuthHandlerFound:
        fail("Upload was requested but could not connect to S3.")
        upload = True

    if upload is True:
        all_examples = get_all_examples()

        for example, _ in all_examples:
            example_path = relpath(no_ext(example), example_dir)
            s3_path = join(__version__, example_path)

            test_png, _, diff_png = get_example_pngs(example)

            if test_png:
                if isfile(test_png):
                    _upload_image(bucket, test_png, s3_path + ".png")
            if diff_png:
                if isfile(diff_png):
                    _upload_image(bucket, diff_png, s3_path + "-diff.png")

def deal_with_output_cells(example):
    output_cells = pytest.config.option.output_cells

    def load_nb(example):
        with open(example, "r") as f:
            return json.load(f)

    def save_nb(example, nb):
        with open(example, "w") as f:
            json.dump(nb, f, ident=2, sort_keys=True)

    def bad_code_cells(nb):
        for worksheet in nb.get("worksheets", []):
            for cell in worksheet.get("cells", []):
                if cell.get("cell_type") == "code":
                    outputs = cell.get("outputs")

                    if isinstance(outputs, list) and len(outputs) > 0:
                        yield cell, outputs

    def complain(fn):
        fn("%s notebook contains output cells" % example)

    if output_cells == 'complain':
        nb = load_nb(example)

        for _, _ in bad_code_cells(nb):
            complain(fail)
            return False
    elif output_cells == 'remove':
        nb = load_nb(example)
        changes = False

        for cell, _ in bad_code_cells(nb):
            cell["outputs"] = []
            changes = True

        if changes:
            complain(warn)
            save_nb(example, nb)

    return True

def get_path_parts(path):
    parts = []
    while True:
        newpath, tail = split(path)
        parts.append(tail)
        path = newpath
        if tail == 'examples':
            break
    parts.reverse()
    return parts

class Timeout(Exception):
    pass

def make_env():
    env = os.environ.copy()
    env['BOKEH_RESOURCES'] = 'relative'
    env['BOKEH_BROWSER'] = 'none'
    return env

def human_bytes(n):
    """
    Return the number of bytes n in more human readable form.
    """
    if n < 1024:
        return '%d B' % n
    k = n / 1024
    if k < 1024:
        return '%d KB' % round(k)
    m = k / 1024
    if m < 1024:
        return '%.1f MB' % m
    g = m / 1024
    return '%.2f GB' % g
<commit_msg>Try and print out more<commit_after>import boto
import json
import os
import pytest

from boto.s3.key import Key as S3Key
from boto.exception import NoAuthHandlerFound
from os.path import split, splitext, abspath, isfile, join, relpath

from ..utils import warn, fail, write, green, ok, red
from ..constants import __version__, s3, s3_bucket, example_dir, build_id

from .collect_examples import get_all_examples

def no_ext(path):
    return splitext(abspath(path))[0]

def get_example_pngs(example_file):
    diff = pytest.config.option.diff
    example_path = no_ext(example_file)
    test_png = "%s-%s-%s.png" % (example_path, __version__, build_id)
    if diff:
        ref_png = "%s-%s-%s.png" % (example_path, diff, build_id)
        diff_png = "%s-%s-%s-diff-%s.png" % (example_path, __version__, diff, build_id)
    else:
        ref_png = None
        diff_png = None
    return (test_png, ref_png, diff_png)

def _upload_image(bucket, path, s3_png_file):
    with open(path, 'rb') as f:
        png = f.read()
    write("%s Uploading image to S3 to %s/%s" % (green(">>>"), s3, s3_png_file))
    key = S3Key(bucket, s3_png_file)
    key.set_metadata("Content-Type", "image/png")
    key.set_contents_from_string(png, policy="public-read")

def upload_example_pngs_to_s3():
    # Test connection
    try:
        conn = boto.connect_s3()
        bucket = conn.get_bucket(s3_bucket)
        upload = True
    except NoAuthHandlerFound:
        fail("Upload was requested but could not connect to S3.")
        upload = True

    if upload is True:
        all_examples = get_all_examples()

        for example, _ in all_examples:
            example_path = relpath(no_ext(example), example_dir)
            s3_path = join(__version__, example_path)

            test_png, _, diff_png = get_example_pngs(example)

            warn(example)
            if test_png:
                write(red("test_png is not none %s" % test_png))
                if isfile(test_png):
                    ok("test_png is file")
                    _upload_image(bucket, test_png, s3_path + ".png")
            if diff_png:
                write(red("diff_png is not none %s" % diff_png))
                if isfile(diff_png):
                    ok("diff_png is file")
                    _upload_image(bucket, diff_png, s3_path + "-diff.png")

def deal_with_output_cells(example):
    output_cells = pytest.config.option.output_cells

    def load_nb(example):
        with open(example, "r") as f:
            return json.load(f)

    def save_nb(example, nb):
        with open(example, "w") as f:
            json.dump(nb, f, ident=2, sort_keys=True)

    def bad_code_cells(nb):
        for worksheet in nb.get("worksheets", []):
            for cell in worksheet.get("cells", []):
                if cell.get("cell_type") == "code":
                    outputs = cell.get("outputs")

                    if isinstance(outputs, list) and len(outputs) > 0:
                        yield cell, outputs

    def complain(fn):
        fn("%s notebook contains output cells" % example)

    if output_cells == 'complain':
        nb = load_nb(example)

        for _, _ in bad_code_cells(nb):
            complain(fail)
            return False
    elif output_cells == 'remove':
        nb = load_nb(example)
        changes = False

        for cell, _ in bad_code_cells(nb):
            cell["outputs"] = []
            changes = True

        if changes:
            complain(warn)
            save_nb(example, nb)

    return True

def get_path_parts(path):
    parts = []
    while True:
        newpath, tail = split(path)
        parts.append(tail)
        path = newpath
        if tail == 'examples':
            break
    parts.reverse()
    return parts

class Timeout(Exception):
    pass

def make_env():
    env = os.environ.copy()
    env['BOKEH_RESOURCES'] = 'relative'
    env['BOKEH_BROWSER'] = 'none'
    return env

def human_bytes(n):
    """
    Return the number of bytes n in more human readable form.
    """
    if n < 1024:
        return '%d B' % n
    k = n / 1024
    if k < 1024:
        return '%d KB' % round(k)
    m = k / 1024
    if m < 1024:
        return '%.1f MB' % m
    g = m / 1024
    return '%.2f GB' % g
<|endoftext|>


<commit_before># Copyright (C) 2021 Rafael Leira, Naudit HPCN S.L.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License,
# version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA  02110-1301, USA.
#
################################################################

import argparse
import json
import os
import sys

from pySMART import Device

from .smartctlfile import SmartctlFile

def main():

    parser = argparse.ArgumentParser(
        description='Generate device.json from data stored in file for future tests.')
    parser.add_argument('--folder', required=True,
                        help='The folder where the device info is stored')
    parser.add_argument('--device', required=True,
                        help='The device name')
    parser.add_argument('--interface', default=None,
                        help='The device interface')

    args = parser.parse_args()

    folder = args.folder
    device_name = args.device
    interface_name = args.interface

    sf = SmartctlFile(folder)

    json_dict = {"name": device_name}

    if interface_name is None:
        dev = Device(device_name, smartctl=sf)

    else:
        dev = Device(device_name, interface=interface_name, smartctl=sf)
        json_dict['interface'] = interface_name

    json_dict['values'] = vars(dev)

    # Remove non serializable objects
    to_delete = [
        'smartctl',
    ]

    # add to list private objects
    for entry in json_dict['values']:
        if entry[0] == '_':
            to_delete.append(entry)

    for todel in to_delete:
        if todel in json_dict['values']:
            del json_dict['values'][todel]

    # Transform attributes
    if 'attributes' in json_dict['values']:
        att_list = []
        for att in json_dict['values']['attributes']:
            if att is None:
                att_list.append(None)
            else:
                att_list.append(vars(att))

        json_dict['values']['attributes'] = att_list

    # Transform tests
    if 'tests' in json_dict['values']:
        test_list = []
        for tst in json_dict['values']['tests']:
            if tst is None:
                test_list.append(None)
            else:
                test_list.append(vars(tst))

        json_dict['values']['tests'] = test_list

    with open(os.path.join(folder, 'device.json'), "w") as f:
        f.write(json.dumps(json_dict, indent=4))

if __name__ == "__main__":
    main()
<commit_msg>gen_test: include device properties<commit_after># Copyright (C) 2021 Rafael Leira, Naudit HPCN S.L.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License,
# version 2, as published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
# MA  02110-1301, USA.
#
################################################################

import argparse
import json
import os
from typing import Dict, Any

from pySMART import Device

from .smartctlfile import SmartctlFile

def get_all_properties(obj) -> Dict[str, Any]:
    type_name = type(obj).__name__
    prop_names = dir(obj)

    ret = vars(obj)

    available_types = ['dict', 'str', 'int', 'float', 'list', 'NoneType']

    for prop_name in prop_names:
        prop_val = getattr(obj, prop_name)
        prop_val_type_name = type(prop_val).__name__

        if prop_name[0] != '_' and prop_val_type_name in available_types and prop_name not in ret:
            ret[prop_name] = prop_val

    return ret

def main():

    parser = argparse.ArgumentParser(
        description='Generate device.json from data stored in file for future tests.')
    parser.add_argument('--folder', required=True,
                        help='The folder where the device info is stored')
    parser.add_argument('--device', required=True,
                        help='The device name')
    parser.add_argument('--interface', default=None,
                        help='The device interface')

    args = parser.parse_args()

    folder = args.folder
    device_name = args.device
    interface_name = args.interface

    sf = SmartctlFile(folder)

    json_dict = {"name": device_name}

    if interface_name is None:
        dev = Device(device_name, smartctl=sf)

    else:
        dev = Device(device_name, interface=interface_name, smartctl=sf)
        json_dict['interface'] = interface_name

    json_dict['values'] = get_all_properties(dev)

    # Remove non serializable objects
    to_delete = [
        'smartctl',
    ]

    # add to list private objects
    for entry in json_dict['values']:
        if entry[0] == '_':
            to_delete.append(entry)

    for todel in to_delete:
        if todel in json_dict['values']:
            del json_dict['values'][todel]

    # Transform attributes
    if 'attributes' in json_dict['values']:
        att_list = []
        for att in json_dict['values']['attributes']:
            if att is None:
                att_list.append(None)
            else:
                att_list.append(vars(att))

        json_dict['values']['attributes'] = att_list

    # Transform tests
    if 'tests' in json_dict['values']:
        test_list = []
        for tst in json_dict['values']['tests']:
            if tst is None:
                test_list.append(None)
            else:
                test_list.append(vars(tst))

        json_dict['values']['tests'] = test_list

    with open(os.path.join(folder, 'device.json'), "w") as f:
        f.write(json.dumps(json_dict, indent=4))

if __name__ == "__main__":
    main()
<|endoftext|>


<commit_before>#! /usr/bin/python
# -*- coding: utf-8 -*-

# import funkcí z jiného adresáře
import os.path

path_to_script = os.path.dirname(os.path.abspath(__file__))
import unittest
from nose.plugins.attrib import attr
import numpy as np

import logging
logger = logging.getLogger(__name__)

from imtools.tree_processing import TreeGenerator
import imtools.surface_measurement as sm

def join_sdp(datadir):
    return os.path.join('./sample_data', datadir)

class HistologyTest(unittest.TestCase):
    interactiveTests = False

    @attr("LAR")
    def test_vessel_tree_lar(self):
        import imtools.gt_lar
        tvg = TreeGenerator(imtools.gt_lar.GTLar)
        yaml_path = os.path.join(path_to_script, "./hist_stats_test.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [100, 100, 100]
        output = tvg.generateTree() # noqa
        if self.interactiveTests:
            tvg.show()

    def test_import_new_vt_format(self):

        tvg = TreeGenerator()
        yaml_path = os.path.join(path_to_script, "vt_biodur.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [150, 150, 150]
        data3d = tvg.generateTree()

    def test_test_export_to_esofspy(self):
        """
        tests export function
        """

        import imtools.vesseltree_export as vt
        yaml_input = os.path.join(path_to_script, "vt_biodur.yaml")
        yaml_output = os.path.join(path_to_script, "delme_esofspy.txt")
        vt.vt2esofspy(yaml_input, yaml_output)

    def test_generate_sample_data(self):
        """
        Test has no strong part
        """

        import lisa.histology_analyser
        lisa.histology_analyser.generate_sample_data()

    @attr("actual")
    def test_surface_density_gensei_data(self):
        import io3d
        dr = io3d.datareader.DataReader()
        datap = dr.Get3DData(join_sdp('gensei_slices/'),
                             dataplus_format=True)
        # total object volume fraction:           0.081000
        # total object volume [(mm)^3]:           81.000000
        # total object surface fraction [1/(mm)]: 0.306450
        # total object surface [(mm)^2]:          306.449981
        segmentation = (datap['data3d'] > 100).astype(np.int8)
        voxelsize_mm = [0.2, 0.2, 0.2]
        volume = np.sum(segmentation) * np.prod(voxelsize_mm)

        Sv = sm.surface_density(segmentation, voxelsize_mm)
        self.assertGreater(volume, 80)
        self.assertLess(volume, 85)
        self.assertGreater(Sv, 0.3)
        self.assertLess(Sv, 0.4)

    def test_surface_measurement(self):

# box
        data1 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1

        Sv1 = sm.surface_density(data1, voxelsize_mm)

# box without small box on corner
        data2 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data2[10:20, 10:20, 10:20] = 1
        data2[10:15, 10:15, 10:15] = 0
        Sv2 = sm.surface_density(data2, voxelsize_mm)

        self.assertEqual(Sv2, Sv1)

# box with hole in one edge
        data3 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data3[10:20, 10:20, 10:20] = 1
        data3[13:18, 13:18, 10:15] = 0
        Sv3 = sm.surface_density(data3, voxelsize_mm)
        self.assertGreater(Sv3, Sv1)
        # import sed3
        # ed = sed3.sed3(im_edg)
        # ed.show()

    def test_surface_measurement_voxelsize_mm(self):
        import scipy

# data 1
        data1 = np.zeros([30, 40, 55])
        voxelsize_mm1 = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1
        data1[13:18, 13:18, 10:15] = 0
# data 2
        voxelsize_mm2 = [0.1, 0.2, 0.3]
        data2 = scipy.ndimage.interpolation.zoom(
            data1,
            zoom=1.0/np.asarray(voxelsize_mm2),
            order=0
        )
        # import sed3
        # ed = sed3.sed3(data1)
        # ed.show()
        # ed = sed3.sed3(data2)
        # ed.show()

        Sv1 = sm.surface_density(data1, voxelsize_mm1)
        Sv2 = sm.surface_density(data2, voxelsize_mm2)
        self.assertGreater(Sv1, Sv2*0.9)
        self.assertLess(Sv1, Sv2*1.1)

    def test_surface_measurement_use_aoi(self):
        """
        Test of AOI. In Sv2 is AOI half in compare with Sv1.
        Sv1 should be half of Sv2
        """
        data1 = np.zeros([30, 60, 60])
        aoi = np.zeros([30, 60, 60])
        aoi[:30, :60, :30] = 1
        voxelsize_mm = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1
        data1[13:18, 13:18, 10:15] = 0

        Sv1 = sm.surface_density(data1, voxelsize_mm, aoi=None)
        Sv2 = sm.surface_density(data1, voxelsize_mm, aoi=aoi)
        self.assertGreater(2*Sv1, Sv2*0.9)
        self.assertLess(2*Sv1, Sv2*1.1)

    def test_surface_measurement_find_edge(self):
        tvg = TreeGenerator()
        yaml_path = os.path.join(path_to_script, "./hist_stats_test.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [100, 100, 100]
        data3d = tvg.generateTree()

        # init histology Analyser
        # metadata = {'voxelsize_mm': tvg.voxelsize_mm}
        # data3d = data3d * 10
        # threshold = 2.5

        im_edg = sm.find_edge(data3d, 0)
        # in this area should be positive edge
        self.assertGreater(
            np.sum(im_edg[25:30, 25:30, 30]),
            3
        )
        # self.assert(im_edg
        # import sed3
        # ed = sed3.sed3(im_edg)
        # ed.show()

if __name__ == "__main__":
    unittest.main()
<commit_msg>sample data path fix<commit_after>#! /usr/bin/python
# -*- coding: utf-8 -*-

# import funkcí z jiného adresáře
import os.path

path_to_script = os.path.dirname(os.path.abspath(__file__))
import unittest
from nose.plugins.attrib import attr
import numpy as np

import logging
logger = logging.getLogger(__name__)

from imtools.tree_processing import TreeGenerator
import imtools.surface_measurement as sm

def join_sdp(datadir):

    return os.path.join(path_to_script, '../sample_data', datadir)

class HistologyTest(unittest.TestCase):
    interactiveTests = False

    @attr("LAR")
    def test_vessel_tree_lar(self):
        import imtools.gt_lar
        tvg = TreeGenerator(imtools.gt_lar.GTLar)
        yaml_path = os.path.join(path_to_script, "./hist_stats_test.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [100, 100, 100]
        output = tvg.generateTree() # noqa
        if self.interactiveTests:
            tvg.show()

    def test_import_new_vt_format(self):

        tvg = TreeGenerator()
        yaml_path = os.path.join(path_to_script, "vt_biodur.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [150, 150, 150]
        data3d = tvg.generateTree()

    def test_test_export_to_esofspy(self):
        """
        tests export function
        """

        import imtools.vesseltree_export as vt
        yaml_input = os.path.join(path_to_script, "vt_biodur.yaml")
        yaml_output = os.path.join(path_to_script, "delme_esofspy.txt")
        vt.vt2esofspy(yaml_input, yaml_output)

    @attr("actual")
    def test_surface_density_gensei_data(self):
        import io3d
        dr = io3d.datareader.DataReader()
        datap = dr.Get3DData(join_sdp('gensei_slices/'),
                             dataplus_format=True)
        # total object volume fraction:           0.081000
        # total object volume [(mm)^3]:           81.000000
        # total object surface fraction [1/(mm)]: 0.306450
        # total object surface [(mm)^2]:          306.449981
        segmentation = (datap['data3d'] > 100).astype(np.int8)
        voxelsize_mm = [0.2, 0.2, 0.2]
        volume = np.sum(segmentation) * np.prod(voxelsize_mm)

        Sv = sm.surface_density(segmentation, voxelsize_mm)
        self.assertGreater(volume, 80)
        self.assertLess(volume, 85)
        self.assertGreater(Sv, 0.3)
        self.assertLess(Sv, 0.4)

    def test_surface_measurement(self):

# box
        data1 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1

        Sv1 = sm.surface_density(data1, voxelsize_mm)

# box without small box on corner
        data2 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data2[10:20, 10:20, 10:20] = 1
        data2[10:15, 10:15, 10:15] = 0
        Sv2 = sm.surface_density(data2, voxelsize_mm)

        self.assertEqual(Sv2, Sv1)

# box with hole in one edge
        data3 = np.zeros([30, 30, 30])
        voxelsize_mm = [1, 1, 1]
        data3[10:20, 10:20, 10:20] = 1
        data3[13:18, 13:18, 10:15] = 0
        Sv3 = sm.surface_density(data3, voxelsize_mm)
        self.assertGreater(Sv3, Sv1)
        # import sed3
        # ed = sed3.sed3(im_edg)
        # ed.show()

    def test_surface_measurement_voxelsize_mm(self):
        import scipy

# data 1
        data1 = np.zeros([30, 40, 55])
        voxelsize_mm1 = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1
        data1[13:18, 13:18, 10:15] = 0
# data 2
        voxelsize_mm2 = [0.1, 0.2, 0.3]
        data2 = scipy.ndimage.interpolation.zoom(
            data1,
            zoom=1.0/np.asarray(voxelsize_mm2),
            order=0
        )
        # import sed3
        # ed = sed3.sed3(data1)
        # ed.show()
        # ed = sed3.sed3(data2)
        # ed.show()

        Sv1 = sm.surface_density(data1, voxelsize_mm1)
        Sv2 = sm.surface_density(data2, voxelsize_mm2)
        self.assertGreater(Sv1, Sv2*0.9)
        self.assertLess(Sv1, Sv2*1.1)

    def test_surface_measurement_use_aoi(self):
        """
        Test of AOI. In Sv2 is AOI half in compare with Sv1.
        Sv1 should be half of Sv2
        """
        data1 = np.zeros([30, 60, 60])
        aoi = np.zeros([30, 60, 60])
        aoi[:30, :60, :30] = 1
        voxelsize_mm = [1, 1, 1]
        data1[10:20, 10:20, 10:20] = 1
        data1[13:18, 13:18, 10:15] = 0

        Sv1 = sm.surface_density(data1, voxelsize_mm, aoi=None)
        Sv2 = sm.surface_density(data1, voxelsize_mm, aoi=aoi)
        self.assertGreater(2*Sv1, Sv2*0.9)
        self.assertLess(2*Sv1, Sv2*1.1)

    def test_surface_measurement_find_edge(self):
        tvg = TreeGenerator()
        yaml_path = os.path.join(path_to_script, "./hist_stats_test.yaml")
        tvg.importFromYaml(yaml_path)
        tvg.voxelsize_mm = [1, 1, 1]
        tvg.shape = [100, 100, 100]
        data3d = tvg.generateTree()

        # init histology Analyser
        # metadata = {'voxelsize_mm': tvg.voxelsize_mm}
        # data3d = data3d * 10
        # threshold = 2.5

        im_edg = sm.find_edge(data3d, 0)
        # in this area should be positive edge
        self.assertGreater(
            np.sum(im_edg[25:30, 25:30, 30]),
            3
        )
        # self.assert(im_edg
        # import sed3
        # ed = sed3.sed3(im_edg)
        # ed.show()

if __name__ == "__main__":
    unittest.main()
<|endoftext|>


<commit_before>import unittest
import aiozmq

class ZmqTransportTests(unittest.TestCase):

    def test_interface(self):
        tr = aiozmq.ZmqTransport()
        self.assertRaises(NotImplementedError, tr.write, [b'data'])
        self.assertRaises(NotImplementedError, tr.abort)
        self.assertRaises(NotImplementedError, tr.getsockopt, 1)
        self.assertRaises(NotImplementedError, tr.setsockopt, 1, 2)
        self.assertRaises(NotImplementedError, tr.set_write_buffer_limits)
        self.assertRaises(NotImplementedError, tr.get_write_buffer_size)
        self.assertRaises(NotImplementedError, tr.bind, 'endpoint')
        self.assertRaises(NotImplementedError, tr.unbind, 'endpoint')
        self.assertRaises(NotImplementedError, tr.bindings)
        self.assertRaises(NotImplementedError, tr.connect, 'endpoint')
        self.assertRaises(NotImplementedError, tr.disconnect, 'endpoint')
        self.assertRaises(NotImplementedError, tr.connections)
        self.assertRaises(NotImplementedError, tr.subscribe, b'filter')
        self.assertRaises(NotImplementedError, tr.unsubscribe, b'filter')
        self.assertRaises(NotImplementedError, tr.subscriptions)

class ZmqProtocolTests(unittest.TestCase):

    def test_interface(self):
        pr = aiozmq.ZmqProtocol()
        self.assertIsNone(pr.msg_received((b'data',)))
<commit_msg>Add missing tests for interfaces<commit_after>import unittest
import aiozmq

class ZmqTransportTests(unittest.TestCase):

    def test_interface(self):
        tr = aiozmq.ZmqTransport()
        self.assertRaises(NotImplementedError, tr.write, [b'data'])
        self.assertRaises(NotImplementedError, tr.abort)
        self.assertRaises(NotImplementedError, tr.getsockopt, 1)
        self.assertRaises(NotImplementedError, tr.setsockopt, 1, 2)
        self.assertRaises(NotImplementedError, tr.set_write_buffer_limits)
        self.assertRaises(NotImplementedError, tr.get_write_buffer_size)
        self.assertRaises(NotImplementedError, tr.pause_reading)
        self.assertRaises(NotImplementedError, tr.resume_reading)
        self.assertRaises(NotImplementedError, tr.bind, 'endpoint')
        self.assertRaises(NotImplementedError, tr.unbind, 'endpoint')
        self.assertRaises(NotImplementedError, tr.bindings)
        self.assertRaises(NotImplementedError, tr.connect, 'endpoint')
        self.assertRaises(NotImplementedError, tr.disconnect, 'endpoint')
        self.assertRaises(NotImplementedError, tr.connections)
        self.assertRaises(NotImplementedError, tr.subscribe, b'filter')
        self.assertRaises(NotImplementedError, tr.unsubscribe, b'filter')
        self.assertRaises(NotImplementedError, tr.subscriptions)

class ZmqProtocolTests(unittest.TestCase):

    def test_interface(self):
        pr = aiozmq.ZmqProtocol()
        self.assertIsNone(pr.msg_received((b'data',)))
<|endoftext|>


<commit_before>from base import HydraTestCase, daemon_version, uv_version

class HydraVersionMethodTestCase(HydraTestCase):
    def test_version(self):
        response = self.makeRequest('version', {})
        self.assertEqual(response['data'], {'version': daemon_version})

        # Check versions
        self.assertEqual(response['data']['version'], daemon_version)
        self.assertEqual(response['data']['deps']['uv_version'], uv_version)
<commit_msg>Fix test<commit_after>from base import HydraTestCase, daemon_version, uv_version

class HydraVersionMethodTestCase(HydraTestCase):
    def test_version(self):
        response = self.makeRequest('version', {})

        # Check versions
        self.assertEqual(response['data']['version'], daemon_version)
        self.assertEqual(response['data']['deps']['uv_version'], uv_version)
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-

from django.http import Http404, HttpResponseNotAllowed
from djclsview import View

__all__ = ['Resource']

METHOD_ACTION_MAP = {
    'singular': {
        'show': {
            'GET': 'show',
            'POST': 'create',
            'PUT': 'update',
            'DELETE': 'destroy',
        },
        'new': {'GET': 'new'},
        'edit': {'GET': 'edit'},
    },

    'plural': {
        'index': {
            'GET': 'index',
            'POST': 'create'
        },
        'show': {
            'GET': 'show',
            'PUT': 'update',
            'POST': 'update',
            'DELETE': 'destroy',
        },
        'new': {'GET': 'new'},
        'edit': {'GET': 'edit'},
    }
}

class Resource(View):

    def __init__(self, request, *args, **params):
        self.request = request
        self.args = args
        self.params = params
        self._called_yet = False

    def __call__(self):
        """Dispatch to an action based on HTTP method + URL."""

        if self._called_yet:
            return self
        self._called_yet = True
        method = self.request.POST.get('_method', self.request.method).upper()
        url_action = self.params.pop('action')
        mode = self.params.pop('mode', None)
        return self._route(method, url_action, mode=mode)()

    def _route(self, method, url_action, mode=None):

        """
        Resolve a method, URL action and (optionally) mode to a 0-ary callable.

        The default behaviour is to map the URL action and the method via the
        `METHOD_ACTION_MAP` to an action on this resource.
        """

        try:
            action_name = METHOD_ACTION_MAP[mode][url_action][method]
        except KeyError:
            # Assume the developer knows what he/she is doing.
            action_name = url_action

        if hasattr(self, action_name):
            return getattr(self, action_name)
        elif mode:
            return lambda: self._not_allowed(mode, url_action)
        else:
            return self._not_found

    def _not_allowed(self, mode, url_action):
        """Return a `HttpResponseNotAllowed` for this mode and URL action."""

        allowed_methods = []
        for meth, action_name in METHOD_ACTION_MAP[mode][url_action].items():
            if hasattr(self, action_name):
                allowed_methods.append(meth)
        return HttpResponseNotAllowed(allowed_methods)

    def _not_found(self):
        raise Http404

    def _format(self):
        """Return a mimetype shortcode, in case there's no Accept header."""

        return self.request.GET.get('format')
<commit_msg>Resource._format() should respect 'format' URL kwargs.<commit_after># -*- coding: utf-8 -*-

from django.http import Http404, HttpResponseNotAllowed
from djclsview import View

__all__ = ['Resource']

METHOD_ACTION_MAP = {
    'singular': {
        'show': {
            'GET': 'show',
            'POST': 'create',
            'PUT': 'update',
            'DELETE': 'destroy',
        },
        'new': {'GET': 'new'},
        'edit': {'GET': 'edit'},
    },

    'plural': {
        'index': {
            'GET': 'index',
            'POST': 'create'
        },
        'show': {
            'GET': 'show',
            'PUT': 'update',
            'POST': 'update',
            'DELETE': 'destroy',
        },
        'new': {'GET': 'new'},
        'edit': {'GET': 'edit'},
    }
}

class Resource(View):

    def __init__(self, request, *args, **params):
        self.request = request
        self.args = args
        self.params = params
        self._called_yet = False

    def __call__(self):
        """Dispatch to an action based on HTTP method + URL."""

        if self._called_yet:
            return self
        self._called_yet = True
        method = self.request.POST.get('_method', self.request.method).upper()
        url_action = self.params.pop('action')
        mode = self.params.pop('mode', None)
        return self._route(method, url_action, mode=mode)()

    def _route(self, method, url_action, mode=None):

        """
        Resolve a method, URL action and (optionally) mode to a 0-ary callable.

        The default behaviour is to map the URL action and the method via the
        `METHOD_ACTION_MAP` to an action on this resource.
        """

        try:
            action_name = METHOD_ACTION_MAP[mode][url_action][method]
        except KeyError:
            # Assume the developer knows what he/she is doing.
            action_name = url_action

        if hasattr(self, action_name):
            return getattr(self, action_name)
        elif mode:
            return lambda: self._not_allowed(mode, url_action)
        else:
            return self._not_found

    def _not_allowed(self, mode, url_action):
        """Return a `HttpResponseNotAllowed` for this mode and URL action."""

        allowed_methods = []
        for meth, action_name in METHOD_ACTION_MAP[mode][url_action].items():
            if hasattr(self, action_name):
                allowed_methods.append(meth)
        return HttpResponseNotAllowed(allowed_methods)

    def _not_found(self):
        raise Http404

    def _format(self):
        """Return a mimetype shortcode, in case there's no Accept header."""

        if self.params.get('format'):
            return self.params['format'].lstrip('.')
        return self.request.GET.get('format')
<|endoftext|>


<commit_before># Copyright (C) 2013-2014, Quentin "mefyl" Hocquet
#
# This software is provided "as is" without warranty of any kind,
# either expressed or implied, including but not limited to the
# implied warranties of fitness for a particular purpose.
#
# See the LICENSE file for more information.

import drake
import os
import subprocess

class Valgrind:

  def __init__(self, path = None):
    if path is None:
      path = drake.Path('valgrind')
    elif isinstance(path, Valgrind):
      self.__path = drake.Path(path.__path)
      self.__version = drake.Version(path.__version)
      return
    self.__path = drake.Path(path)
    try:
      output = subprocess.check_output([str(self.path), '--version'])
    except Exception as e:
      raise Exception('Unable to find %s' % self.path) from e
    output = output.decode()
    prefix = 'valgrind-'
    if not output.startswith(prefix):
      raise Exception('Unable to parse valgrind version: %s' % output)
    output = output[len(prefix):]
    self.__version = drake.Version(output)

  @property
  def path(self):
    return self.__path

  @property
  def version(self):
    return self.__version

class ValgrindRunner(drake.Runner):

  def __init__(self, exe, args = None, env = None, valgrind = None):
    super().__init__(exe, args = args, env = env)
    self.__valgrind = Valgrind(valgrind)
    self.__valgrind_status = drake.node(
      '%s.valgrind' % self.executable.name_relative)
    self.__valgrind_status.builder = self
    self.valgrind_reporting = drake.Runner.Reporting.on_failure

  @property
  def command(self):
    return [
      str(self.__valgrind.path),
      '--leak-check=full',
      '--num-callers=50',
      '--log-file=%s' % self.__valgrind_status.path(),
      '--error-exitcode=1',
    ] + super().command

  def _report(self, status):
    super()._report(status)
    if self._must_report(self.valgrind_reporting, status):
      self._report_node(self.__valgrind_status)
<commit_msg>ValgrindRunner: new valgrind_args argument.<commit_after># Copyright (C) 2013-2014, Quentin "mefyl" Hocquet
#
# This software is provided "as is" without warranty of any kind,
# either expressed or implied, including but not limited to the
# implied warranties of fitness for a particular purpose.
#
# See the LICENSE file for more information.

import drake
import os
import subprocess

class Valgrind:

  def __init__(self, path = None):
    if path is None:
      path = drake.Path('valgrind')
    elif isinstance(path, Valgrind):
      self.__path = drake.Path(path.__path)
      self.__version = drake.Version(path.__version)
      return
    self.__path = drake.Path(path)
    try:
      output = subprocess.check_output([str(self.path), '--version'])
    except Exception as e:
      raise Exception('Unable to find %s' % self.path) from e
    output = output.decode()
    prefix = 'valgrind-'
    if not output.startswith(prefix):
      raise Exception('Unable to parse valgrind version: %s' % output)
    output = output[len(prefix):]
    self.__version = drake.Version(output)

  @property
  def path(self):
    return self.__path

  @property
  def version(self):
    return self.__version

class ValgrindRunner(drake.Runner):

  def __init__(self, exe, args = None, env = None, valgrind = None, valgrind_args = None):
    super().__init__(exe, args = args, env = env)
    self.__valgrind = Valgrind(valgrind)
    self.__valgrind_status = drake.node(
      '%s.valgrind' % self.executable.name_relative)
    self.__valgrind_status.builder = self
    self.valgrind_reporting = drake.Runner.Reporting.on_failure
    self.__valgrind_args = valgrind_args or []

  @property
  def command(self):
    return [
      str(self.__valgrind.path),
      '--leak-check=full',
      '--num-callers=50',
      '--log-file=%s' % self.__valgrind_status.path(),
      '--error-exitcode=1',
    ] + self.__valgrind_args + super().command

  def _report(self, status):
    super()._report(status)
    if self._must_report(self.valgrind_reporting, status):
      self._report_node(self.__valgrind_status)
<|endoftext|>


<commit_before>###
# Copyright (c) 2002-2004, Jeremiah Fincher
# Copyright (c) 2010, James Vega
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Contains simple socket drivers.  Asyncore bugged (haha, pun!) me.
"""

from __future__ import division

import sys
import time
import select
import socket
try:
    import ssl
    SSLError = ssl.SSLError
except:
    drivers.log.debug('ssl module is not available, '
                      'cannot connect to SSL servers.')
    class SSLError(Exception):
        pass

import supybot.log as log
import supybot.conf as conf
import supybot.utils as utils
import supybot.world as world
import supybot.drivers as drivers
import supybot.schedule as schedule
from itertools import imap

class SocketDriver(drivers.IrcDriver, drivers.ServersMixin):
    def __init__(self, irc):
        self.irc = irc
        drivers.IrcDriver.__init__(self, irc)
        drivers.ServersMixin.__init__(self, irc)
        self.conn = None
        self.servers = ()
        self.eagains = 0
        self.inbuffer = b''
        self.outbuffer = ''
        self.zombie = False
        self.connected = False
        self.writeCheckTime = None
        self.nextReconnectTime = None
        self.resetDelay()
        if self.networkGroup.get('ssl').value and not globals().has_key('ssl'):
            drivers.log.error('The Socket driver can not connect to SSL '
                              'servers for your Python version.  Try the '
                              'Twisted driver instead, or install a Python'
                              'version that supports SSL (2.6 and greater).')
        else:
            self.connect()

    def getDelay(self):
        ret = self.currentDelay
        self.currentDelay = min(self.currentDelay * 2,
                                conf.supybot.drivers.maxReconnectWait())
        return ret

    def resetDelay(self):
        self.currentDelay = 10.0

    def _getNextServer(self):
        oldServer = getattr(self, 'currentServer', None)
        server = drivers.ServersMixin._getNextServer(self)
        if self.currentServer != oldServer:
            self.resetDelay()
        return server

    def _handleSocketError(self, e):
        # (11, 'Resource temporarily unavailable') raised if connect
        # hasn't finished yet.  We'll keep track of how many we get.
        if e.args[0] != 11 or self.eagains > 120:
            drivers.log.disconnect(self.currentServer, e)
            try:
                self.conn.close()
            except:
                pass
            self.connected = False
            self.scheduleReconnect()
        else:
            log.debug('Got EAGAIN, current count: %s.', self.eagains)
            self.eagains += 1

    def _sendIfMsgs(self):
        if not self.zombie:
            msgs = [self.irc.takeMsg()]
            while msgs[-1] is not None:
                msgs.append(self.irc.takeMsg())
            del msgs[-1]
            self.outbuffer += ''.join(imap(str, msgs))
        if self.outbuffer:
            try:
                if sys.version_info[0] < 3:
                    sent = self.conn.send(self.outbuffer)
                else:
                    sent = self.conn.send(self.outbuffer.encode())
                self.outbuffer = self.outbuffer[sent:]
                self.eagains = 0
            except socket.error, e:
                self._handleSocketError(e)
        if self.zombie and not self.outbuffer:
            self._reallyDie()

    def run(self):
        now = time.time()
        if self.nextReconnectTime is not None and now > self.nextReconnectTime:
            self.reconnect()
        elif self.writeCheckTime is not None and now > self.writeCheckTime:
            self._checkAndWriteOrReconnect()
        if not self.connected:
            # We sleep here because otherwise, if we're the only driver, we'll
            # spin at 100% CPU while we're disconnected.
            time.sleep(conf.supybot.drivers.poll())
            return
        self._sendIfMsgs()
        try:
            self.inbuffer += self.conn.recv(1024)
            self.eagains = 0 # If we successfully recv'ed, we can reset this.
            lines = self.inbuffer.split(b'\n')
            self.inbuffer = lines.pop()
            for line in lines:
                if sys.version_info[0] >= 3:
                    line = line.decode(errors='replace')
                msg = drivers.parseMsg(line)
                if msg is not None:
                    self.irc.feedMsg(msg)
        except socket.timeout:
            pass
        except SSLError, e:
            if e.args[0] == 'The read operation timed out':
                pass
            else:
                self._handleSocketError(e)
                return
        except socket.error, e:
            self._handleSocketError(e)
            return
        if not self.irc.zombie:
            self._sendIfMsgs()

    def connect(self, **kwargs):
        self.reconnect(reset=False, **kwargs)

    def reconnect(self, reset=True):
        self.nextReconnectTime = None
        if self.connected:
            drivers.log.reconnect(self.irc.network)
            self.conn.close()
            self.connected = False
        if reset:
            drivers.log.debug('Resetting %s.', self.irc)
            self.irc.reset()
        else:
            drivers.log.debug('Not resetting %s.', self.irc)
        server = self._getNextServer()
        drivers.log.connect(self.currentServer)
        try:
            socks_proxy = getattr(conf.supybot.networks, self.irc.network) \
                    .socksproxy()
            try:
                if socks_proxy:
                    import socks
            except ImportError:
                log.error('Cannot use socks proxy (SocksiPy not installed), '
                        'using direct connection instead.')
                socks_proxy = ''
            self.conn = utils.net.getSocket(server[0], socks_proxy)
            vhost = conf.supybot.protocols.irc.vhost()
            self.conn.bind((vhost, 0))
        except socket.error, e:
            drivers.log.connectError(self.currentServer, e)
            self.scheduleReconnect()
            return
        # We allow more time for the connect here, since it might take longer.
        # At least 10 seconds.
        self.conn.settimeout(max(10, conf.supybot.drivers.poll()*10))
        try:
            self.conn.connect(server)
            self.conn.settimeout(conf.supybot.drivers.poll())
            if getattr(conf.supybot.networks, self.irc.network).ssl():
                assert globals().has_key('ssl')
                self.conn = ssl.wrap_socket(self.conn)
            self.connected = True
            self.resetDelay()
        except socket.error, e:
            if e.args[0] == 115:
                now = time.time()
                when = now + 60
                whenS = log.timestamp(when)
                drivers.log.debug('Connection in progress, scheduling '
                                  'connectedness check for %s', whenS)
                self.writeCheckTime = when
            else:
                drivers.log.connectError(self.currentServer, e)
                self.scheduleReconnect()
            return

    def _checkAndWriteOrReconnect(self):
        self.writeCheckTime = None
        drivers.log.debug('Checking whether we are connected.')
        (_, w, _) = select.select([], [self.conn], [], 0)
        if w:
            drivers.log.debug('Socket is writable, it might be connected.')
            self.connected = True
            self.resetDelay()
        else:
            drivers.log.connectError(self.currentServer, 'Timed out')
            self.reconnect()

    def scheduleReconnect(self):
        when = time.time() + self.getDelay()
        if not world.dying:
            drivers.log.reconnect(self.irc.network, when)
        if self.nextReconnectTime:
            drivers.log.error('Updating next reconnect time when one is '
                              'already present.  This is a bug; please '
                              'report it, with an explanation of what caused '
                              'this to happen.')
        self.nextReconnectTime = when

    def die(self):
        self.zombie = True
        if self.nextReconnectTime is not None:
            self.nextReconnectTime = None
        if self.writeCheckTime is not None:
            self.writeCheckTime = None
        drivers.log.die(self.irc)

    def _reallyDie(self):
        if self.conn is not None:
            self.conn.close()
        drivers.IrcDriver.die(self)
        # self.irc.die() Kill off the ircs yourself, jerk!

    def name(self):
        return '%s(%s)' % (self.__class__.__name__, self.irc)

Driver = SocketDriver

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:

<commit_msg>Socket driver: Apply supybot.drivers.poll as soon as modified (do not require reconnect).<commit_after>###
# Copyright (c) 2002-2004, Jeremiah Fincher
# Copyright (c) 2010, James Vega
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
#
#   * Redistributions of source code must retain the above copyright notice,
#     this list of conditions, and the following disclaimer.
#   * Redistributions in binary form must reproduce the above copyright notice,
#     this list of conditions, and the following disclaimer in the
#     documentation and/or other materials provided with the distribution.
#   * Neither the name of the author of this software nor the name of
#     contributors to this software may be used to endorse or promote products
#     derived from this software without specific prior written consent.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
# POSSIBILITY OF SUCH DAMAGE.
###

"""
Contains simple socket drivers.  Asyncore bugged (haha, pun!) me.
"""

from __future__ import division

import sys
import time
import select
import socket
try:
    import ssl
    SSLError = ssl.SSLError
except:
    drivers.log.debug('ssl module is not available, '
                      'cannot connect to SSL servers.')
    class SSLError(Exception):
        pass

import supybot.log as log
import supybot.conf as conf
import supybot.utils as utils
import supybot.world as world
import supybot.drivers as drivers
import supybot.schedule as schedule
from itertools import imap

class SocketDriver(drivers.IrcDriver, drivers.ServersMixin):
    def __init__(self, irc):
        self.irc = irc
        drivers.IrcDriver.__init__(self, irc)
        drivers.ServersMixin.__init__(self, irc)
        self.conn = None
        self.servers = ()
        self.eagains = 0
        self.inbuffer = b''
        self.outbuffer = ''
        self.zombie = False
        self.connected = False
        self.writeCheckTime = None
        self.nextReconnectTime = None
        self.resetDelay()
        if self.networkGroup.get('ssl').value and not globals().has_key('ssl'):
            drivers.log.error('The Socket driver can not connect to SSL '
                              'servers for your Python version.  Try the '
                              'Twisted driver instead, or install a Python'
                              'version that supports SSL (2.6 and greater).')
        else:
            self.connect()

    def getDelay(self):
        ret = self.currentDelay
        self.currentDelay = min(self.currentDelay * 2,
                                conf.supybot.drivers.maxReconnectWait())
        return ret

    def resetDelay(self):
        self.currentDelay = 10.0

    def _getNextServer(self):
        oldServer = getattr(self, 'currentServer', None)
        server = drivers.ServersMixin._getNextServer(self)
        if self.currentServer != oldServer:
            self.resetDelay()
        return server

    def _handleSocketError(self, e):
        # (11, 'Resource temporarily unavailable') raised if connect
        # hasn't finished yet.  We'll keep track of how many we get.
        if e.args[0] != 11 or self.eagains > 120:
            drivers.log.disconnect(self.currentServer, e)
            try:
                self.conn.close()
            except:
                pass
            self.connected = False
            self.scheduleReconnect()
        else:
            log.debug('Got EAGAIN, current count: %s.', self.eagains)
            self.eagains += 1

    def _sendIfMsgs(self):
        if not self.zombie:
            msgs = [self.irc.takeMsg()]
            while msgs[-1] is not None:
                msgs.append(self.irc.takeMsg())
            del msgs[-1]
            self.outbuffer += ''.join(imap(str, msgs))
        if self.outbuffer:
            try:
                if sys.version_info[0] < 3:
                    sent = self.conn.send(self.outbuffer)
                else:
                    sent = self.conn.send(self.outbuffer.encode())
                self.outbuffer = self.outbuffer[sent:]
                self.eagains = 0
            except socket.error, e:
                self._handleSocketError(e)
        if self.zombie and not self.outbuffer:
            self._reallyDie()

    def run(self):
        now = time.time()
        if self.nextReconnectTime is not None and now > self.nextReconnectTime:
            self.reconnect()
        elif self.writeCheckTime is not None and now > self.writeCheckTime:
            self._checkAndWriteOrReconnect()
        if not self.connected:
            # We sleep here because otherwise, if we're the only driver, we'll
            # spin at 100% CPU while we're disconnected.
            time.sleep(conf.supybot.drivers.poll())
            return
        self._sendIfMsgs()
        try:
            self.inbuffer += self.conn.recv(1024)
            self.eagains = 0 # If we successfully recv'ed, we can reset this.
            lines = self.inbuffer.split(b'\n')
            self.inbuffer = lines.pop()
            for line in lines:
                if sys.version_info[0] >= 3:
                    line = line.decode(errors='replace')
                msg = drivers.parseMsg(line)
                if msg is not None:
                    self.irc.feedMsg(msg)
        except socket.timeout:
            pass
        except SSLError, e:
            if e.args[0] == 'The read operation timed out':
                pass
            else:
                self._handleSocketError(e)
                return
        except socket.error, e:
            self._handleSocketError(e)
            return
        if not self.irc.zombie:
            self._sendIfMsgs()

    def connect(self, **kwargs):
        self.reconnect(reset=False, **kwargs)

    def reconnect(self, reset=True):
        self.nextReconnectTime = None
        if self.connected:
            drivers.log.reconnect(self.irc.network)
            self.conn.close()
            self.connected = False
        if reset:
            drivers.log.debug('Resetting %s.', self.irc)
            self.irc.reset()
        else:
            drivers.log.debug('Not resetting %s.', self.irc)
        server = self._getNextServer()
        drivers.log.connect(self.currentServer)
        try:
            socks_proxy = getattr(conf.supybot.networks, self.irc.network) \
                    .socksproxy()
            try:
                if socks_proxy:
                    import socks
            except ImportError:
                log.error('Cannot use socks proxy (SocksiPy not installed), '
                        'using direct connection instead.')
                socks_proxy = ''
            self.conn = utils.net.getSocket(server[0], socks_proxy)
            vhost = conf.supybot.protocols.irc.vhost()
            self.conn.bind((vhost, 0))
        except socket.error, e:
            drivers.log.connectError(self.currentServer, e)
            self.scheduleReconnect()
            return
        # We allow more time for the connect here, since it might take longer.
        # At least 10 seconds.
        self.conn.settimeout(max(10, conf.supybot.drivers.poll()*10))
        try:
            self.conn.connect(server)
            def setTimeout():
                self.conn.settimeout(conf.supybot.drivers.poll())
            conf.supybot.drivers.poll.addCallback(setTimeout)
            setTimeout()
            if getattr(conf.supybot.networks, self.irc.network).ssl():
                assert globals().has_key('ssl')
                self.conn = ssl.wrap_socket(self.conn)
            self.connected = True
            self.resetDelay()
        except socket.error, e:
            if e.args[0] == 115:
                now = time.time()
                when = now + 60
                whenS = log.timestamp(when)
                drivers.log.debug('Connection in progress, scheduling '
                                  'connectedness check for %s', whenS)
                self.writeCheckTime = when
            else:
                drivers.log.connectError(self.currentServer, e)
                self.scheduleReconnect()
            return

    def _checkAndWriteOrReconnect(self):
        self.writeCheckTime = None
        drivers.log.debug('Checking whether we are connected.')
        (_, w, _) = select.select([], [self.conn], [], 0)
        if w:
            drivers.log.debug('Socket is writable, it might be connected.')
            self.connected = True
            self.resetDelay()
        else:
            drivers.log.connectError(self.currentServer, 'Timed out')
            self.reconnect()

    def scheduleReconnect(self):
        when = time.time() + self.getDelay()
        if not world.dying:
            drivers.log.reconnect(self.irc.network, when)
        if self.nextReconnectTime:
            drivers.log.error('Updating next reconnect time when one is '
                              'already present.  This is a bug; please '
                              'report it, with an explanation of what caused '
                              'this to happen.')
        self.nextReconnectTime = when

    def die(self):
        self.zombie = True
        if self.nextReconnectTime is not None:
            self.nextReconnectTime = None
        if self.writeCheckTime is not None:
            self.writeCheckTime = None
        drivers.log.die(self.irc)

    def _reallyDie(self):
        if self.conn is not None:
            self.conn.close()
        drivers.IrcDriver.die(self)
        # self.irc.die() Kill off the ircs yourself, jerk!

    def name(self):
        return '%s(%s)' % (self.__class__.__name__, self.irc)

Driver = SocketDriver

# vim:set shiftwidth=4 softtabstop=4 expandtab textwidth=79:

<|endoftext|>