<commit_before>#
# Copyright (C) 2012 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, protected-access
from __future__ import absolute_import

import os.path
import unittest

import anyconfig.backend.ini as TT
import anyconfig.tests.common

from anyconfig.compat import OrderedDict as ODict
from anyconfig.mdicts import UpdateWithReplaceDict
from anyconfig.tests.common import dicts_equal

CNF_0_S = """[DEFAULT]
a: 0
b: bbb
c: 5

[sect0]
d: x,y,z
"""

CNF_0 = ODict((("DEFAULT", ODict((("a", "0"), ("b", "bbb"), ("c", "5")))),
               ("sect0", ODict((("a", "0"), ("b", "bbb"), ("c", "5"),
                                ("d", "x,y,z"))))))
CNF_1 = ODict((("DEFAULT", ODict((("a", 0), ("b", "bbb"), ("c", 5)))),
               ("sect0", ODict((("a", 0), ("b", "bbb"), ("c", 5),
                                ("d", "x y z".split()))))))

class Test10(unittest.TestCase):

    cnf = CNF_0
    cnf_s = CNF_0_S
    load_options = dict(allow_no_value=False, defaults=None)
    dump_options = dict()
    is_order_kept = True

    def setUp(self):
        self.psr = TT.Parser()

    def test_10_loads(self):
        cnf = self.psr.loads(self.cnf_s)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_12_loads__w_options(self):
        cnf = self.psr.loads(self.cnf_s, **self.load_options)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dumps(self):
        cnf_s = self.psr.dumps(self.cnf)
        self.assertTrue(cnf_s)
        cnf = self.psr.loads(cnf_s)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_22_dumps__w_options(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf, **self.dump_options))
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_loads_with_order_kept(self):
        cnf = self.psr.loads(self.cnf_s, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(dicts_equal(cnf, self.cnf, ordered=True),
                            "\n %r\nvs.\n %r" % (cnf, self.cnf))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

class Test11(Test10):

    cnf = CNF_1

    def test_10_loads(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_12_loads__w_options(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True,
                             **self.load_options)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dumps(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf), ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_22_dumps__w_options(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf, **self.dump_options),
                             ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_loads_with_order_kept(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(dicts_equal(cnf, self.cnf, ordered=True),
                            "\n %r\nvs.\n %r" % (cnf, self.cnf))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

class Test12(Test10):

    test_10_loads = test_12_loads__w_options = lambda: True
    test_20_loads = test_22_loads__w_options = lambda: True

    def test_10_loads__invalid_input(self):
        invalid_ini = "key=name"
        self.assertRaises(Exception, self.psr.loads, invalid_ini)

class Test20(unittest.TestCase):

    psr_cls = TT.Parser
    cnf = CNF_0
    cnf_s = CNF_0_S
    cnf_fn = "conf0.ini"
    is_order_kept = True

    def setUp(self):
        self.psr = self.psr_cls()
        self.workdir = anyconfig.tests.common.setup_workdir()
        self.cpath = os.path.join(self.workdir, self.cnf_fn)
        with self.psr.wopen(self.cpath) as out:
            out.write(self.cnf_s)

    def tearDown(self):
        anyconfig.tests.common.cleanup_workdir(self.workdir)

    def test_10_load(self):
        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dump(self):
        self.psr.dump(self.cnf, self.cpath)
        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_load__from_stream(self):
        with self.psr.ropen(self.cpath) as strm:
            cnf = self.psr.load(strm)

        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_40_dump__to_stream(self):
        with self.psr.wopen(self.cpath) as strm:
            self.psr.dump(self.cnf, strm)

        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_50_load_with_order_kept(self):
        cnf = self.psr.load(self.cpath, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(list(cnf.keys()), list(self.cnf.keys()))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

# vim:sw=4:ts=4:et:
<commit_msg>enhancement: add a test case to check the format of outputs by INI backend<commit_after>#
# Copyright (C) 2012 - 2015 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, protected-access
from __future__ import absolute_import

import os.path
import unittest

import anyconfig.backend.ini as TT
import anyconfig.tests.common

from anyconfig.compat import OrderedDict as ODict
from anyconfig.mdicts import UpdateWithReplaceDict
from anyconfig.tests.common import dicts_equal

CNF_0_S = """[DEFAULT]
a: 0
b: bbb
c: 5

[sect0]
d: x,y,z
"""

CNF_0 = ODict((("DEFAULT", ODict((("a", "0"), ("b", "bbb"), ("c", "5")))),
               ("sect0", ODict((("a", "0"), ("b", "bbb"), ("c", "5"),
                                ("d", "x,y,z"))))))
CNF_1 = ODict((("DEFAULT", ODict((("a", 0), ("b", "bbb"), ("c", 5)))),
               ("sect0", ODict((("a", 0), ("b", "bbb"), ("c", 5),
                                ("d", "x y z".split()))))))

class Test10(unittest.TestCase):

    cnf = CNF_0
    cnf_s = CNF_0_S
    load_options = dict(allow_no_value=False, defaults=None)
    dump_options = dict()
    is_order_kept = True

    def setUp(self):
        self.psr = TT.Parser()

    def test_10_loads(self):
        cnf = self.psr.loads(self.cnf_s)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_12_loads__w_options(self):
        cnf = self.psr.loads(self.cnf_s, **self.load_options)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dumps(self):
        cnf_s = self.psr.dumps(self.cnf)
        self.assertTrue(cnf_s)
        cnf = self.psr.loads(cnf_s)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_22_dumps__w_options(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf, **self.dump_options))
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_loads_with_order_kept(self):
        cnf = self.psr.loads(self.cnf_s, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(dicts_equal(cnf, self.cnf, ordered=True),
                            "\n %r\nvs.\n %r" % (cnf, self.cnf))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

class Test11(Test10):

    cnf = CNF_1

    def test_10_loads(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_12_loads__w_options(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True,
                             **self.load_options)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dumps(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf), ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_22_dumps__w_options(self):
        cnf = self.psr.loads(self.psr.dumps(self.cnf, **self.dump_options),
                             ac_parse_value=True)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_loads_with_order_kept(self):
        cnf = self.psr.loads(self.cnf_s, ac_parse_value=True, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(dicts_equal(cnf, self.cnf, ordered=True),
                            "\n %r\nvs.\n %r" % (cnf, self.cnf))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

class Test12(Test10):

    test_10_loads = test_12_loads__w_options = lambda: True
    test_20_loads = test_22_loads__w_options = lambda: True

    def test_10_loads__invalid_input(self):
        invalid_ini = "key=name"
        self.assertRaises(Exception, self.psr.loads, invalid_ini)

    def test_20_dumps__format(self):
        ref = self.cnf_s.replace(': ', ' = ')
        self.assertEquals(self.psr.dumps(self.cnf), ref)

class Test20(unittest.TestCase):

    psr_cls = TT.Parser
    cnf = CNF_0
    cnf_s = CNF_0_S
    cnf_fn = "conf0.ini"
    is_order_kept = True

    def setUp(self):
        self.psr = self.psr_cls()
        self.workdir = anyconfig.tests.common.setup_workdir()
        self.cpath = os.path.join(self.workdir, self.cnf_fn)
        with self.psr.wopen(self.cpath) as out:
            out.write(self.cnf_s)

    def tearDown(self):
        anyconfig.tests.common.cleanup_workdir(self.workdir)

    def test_10_load(self):
        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_20_dump(self):
        self.psr.dump(self.cnf, self.cpath)
        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_30_load__from_stream(self):
        with self.psr.ropen(self.cpath) as strm:
            cnf = self.psr.load(strm)

        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_40_dump__to_stream(self):
        with self.psr.wopen(self.cpath) as strm:
            self.psr.dump(self.cnf, strm)

        cnf = self.psr.load(self.cpath)
        self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))
        self.assertTrue(isinstance(cnf, UpdateWithReplaceDict))

    def test_50_load_with_order_kept(self):
        cnf = self.psr.load(self.cpath, ac_ordered=True)
        if self.is_order_kept:
            self.assertTrue(list(cnf.keys()), list(self.cnf.keys()))
        else:
            self.assertTrue(dicts_equal(cnf, self.cnf), str(cnf))

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>"""
WSGI config for fridge project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
"""

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fridge.settings")

from django.core.wsgi import get_wsgi_application
from dj_static import Cling

application = Cling(get_wsgi_application())
<commit_msg>specify prod settings file<commit_after>"""
WSGI config for fridge project.

It exposes the WSGI callable as a module-level variable named ``application``.

For more information on this file, see
https://docs.djangoproject.com/en/1.7/howto/deployment/wsgi/
"""

import os
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "fridge.settings.prod")

from django.core.wsgi import get_wsgi_application
from dj_static import Cling

application = Cling(get_wsgi_application())
<|endoftext|>


<commit_before>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name, protected-access
from __future__ import absolute_import
import unittest

import anyconfig.backend.xml as TT
import anyconfig.backend.tests.ini
import anyconfig.tests.common
import anyconfig.compat

from anyconfig.tests.common import dicts_equal, to_bytes

XML_W_NS_S = """

   1
   <val:c>C</val:c>

"""

class Test_00(unittest.TestCase):

    def test__namespaces_from_file(self):
        ref = {"http://example.com/ns/config": '',
               "http://example.com/ns/config/val": "val"}
        xmlfile = anyconfig.compat.StringIO(XML_W_NS_S)
        self.assertTrue(dicts_equal(TT._namespaces_from_file(xmlfile), ref))

class Test_10(unittest.TestCase):

    def test_elem_to_container__None(self):
        self.assertEqual(TT.elem_to_container(None, dict, {}), dict())

    def test_root_to_container__None(self):
        self.assertEqual(TT.root_to_container(None, dict, {}), dict())

    def test_elem_to_container__attrs(self):
        ref = dict(a={"@attrs": dict(x='1', y='y')})
        root = TT.ET.XML("")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__child(self):
        ref = dict(a=dict(b="b"))
        root = TT.ET.XML("b")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__children(self):
        ref = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        root = TT.ET.XML("b<c>c</c>")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__text(self):
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), {'a': 'A'})

    def test_elem_to_container__text_attrs(self):
        ref = dict(a={"@attrs": {'x': 'X'}, "@text": "A"})
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

def tree_to_string(tree):
    return TT.ET.tostring(tree.getroot())

class Test_20(unittest.TestCase):

    def test_00_container_to_etree__None(self):
        self.assertTrue(TT.container_to_etree(None) is None)

    def test_10_container_to_etree__text_attrs(self):
        ref = to_bytes('A')
        obj = dict(a={"@attrs": {'x': 'X', 'y': 'Y'}, "@text": "A"})
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_20_container_to_etree__child(self):
        ref = to_bytes("b")
        obj = dict(a=dict(b="b"))
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_22_container_to_etree__children(self):
        ref = to_bytes("b<c>c</c>")
        obj = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

# vim:sw=4:ts=4:et:
<commit_msg>fix: revert the previous commit accidentally contains two different changes<commit_after>#
# Copyright (C) 2012 - 2017 Satoru SATOH <ssato @ redhat.com>
# License: MIT
#
# pylint: disable=missing-docstring, invalid-name
from __future__ import absolute_import
import unittest

import anyconfig.backend.xml as TT
import anyconfig.backend.tests.ini
import anyconfig.tests.common
import anyconfig.compat

from anyconfig.tests.common import dicts_equal, to_bytes

XML_W_NS_S = """

   1
   <val:c>C</val:c>

"""

class Test_00(unittest.TestCase):

    def test__namespaces_from_file(self):
        ref = {"http://example.com/ns/config": '',
               "http://example.com/ns/config/val": "val"}
        xmlfile = anyconfig.compat.StringIO(XML_W_NS_S)
        self.assertTrue(dicts_equal(TT._namespaces_from_file(xmlfile), ref))

class Test_10(unittest.TestCase):

    def test_elem_to_container__None(self):
        self.assertEqual(TT.elem_to_container(None, dict, {}), dict())

    def test_root_to_container__None(self):
        self.assertEqual(TT.root_to_container(None, dict, {}), dict())

    def test_elem_to_container__attrs(self):
        ref = dict(a={"@attrs": dict(x='1', y='y')})
        root = TT.ET.XML("")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__child(self):
        ref = dict(a=dict(b="b"))
        root = TT.ET.XML("b")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__children(self):
        ref = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        root = TT.ET.XML("b<c>c</c>")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

    def test_elem_to_container__text(self):
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), {'a': 'A'})

    def test_elem_to_container__text_attrs(self):
        ref = dict(a={"@attrs": {'x': 'X'}, "@text": "A"})
        root = TT.ET.XML("A")
        self.assertEqual(TT.elem_to_container(root, dict, {}), ref)

def tree_to_string(tree):
    return TT.ET.tostring(tree.getroot())

class Test_20(unittest.TestCase):

    def test_00_container_to_etree__None(self):
        self.assertTrue(TT.container_to_etree(None) is None)

    def test_10_container_to_etree__text_attrs(self):
        ref = to_bytes('A')
        obj = dict(a={"@attrs": {'x': 'X', 'y': 'Y'}, "@text": "A"})
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_20_container_to_etree__child(self):
        return  # TODO

        ref = to_bytes("b")
        obj = dict(a=dict(b="b"))
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

    def test_22_container_to_etree__children(self):
        return  # TODO

        ref = to_bytes("b<c>c</c>")
        obj = {'a': {'@children': [{'b': 'b'}, {'c': 'c'}]}}
        res = TT.container_to_etree(obj)
        self.assertEqual(tree_to_string(res), ref)

# vim:sw=4:ts=4:et:
<|endoftext|>


<commit_before>#!/usr/bin/env python3

# Copyright (c) 2017 - present Facebook, Inc.
# All rights reserved.
#
# This source code is licensed under the BSD style license found in the
# LICENSE file in the root directory of this source tree. An additional grant
# of patent rights can be found in the PATENTS file in the same directory.

import json
import os
import sys

REPORT_FILENAME = sys.argv[1]

def load_report():
    with open(REPORT_FILENAME, 'r') as file_in:
        return json.load(file_in)

def compute_duplicates(report):
    table = {}
    for e in report:
        bug_hash = e['hash']
        if bug_hash in table:
            table[bug_hash].append(e)
        else:
            table[bug_hash] = [e]
    duplicates = {}
    for key, value in table.items():
        if len(value) >= 1:
            duplicates[key] = value
    return duplicates

def compute_max_hash(duplicates):
    max_length = 0
    max_hash = 0
    for key, value in duplicates.items():
        length = len(value)
        if length > max_length:
            max_length = length
            max_hash = key
    return max_hash

def print_duplicates(duplicates):
    with open('duplicates.txt', 'w') as file_out:
        for key, value in duplicates.items():
            reports = set([(e['bug_type'], e['qualifier']) for e in value])
            messages = ['{} {}'.format(bug, msg) for (bug, msg) in reports]
            if len(messages) > 1:
                file_out.write('\nWith bug hash {}:\n\t{}\n'.format(
                    key,
                    '\n\t'.join(messages)))

report = load_report()
duplicates = compute_duplicates(report)
print_duplicates(duplicates)
max_hash = compute_max_hash(duplicates)
print('max hash: {}'.format(max_hash))
duplicates = [e for e in report if e['hash'] == max_hash]
with open('duplicates.json', 'w') as file_out:
    json.dump(duplicates, file_out, indent=2, separators=(',', ': '))
<commit_msg>[infer] remove file committed by mistake<commit_after><|endoftext|>


<commit_before>
import unittest
import numpy
from apgl.util.Sampling import Sampling

class  SamplingTest(unittest.TestCase):
    def testCrossValidation(self):
        numExamples = 10
        folds = 2

        indices = Sampling.crossValidation(folds, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([5, 6, 7, 8, 9], [0, 1, 2, 3, 4]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 3, 4], [5, 6, 7, 8, 9]))

        indices = Sampling.crossValidation(3, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([3, 4, 5, 6, 7, 8, 9], [0, 1, 2]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 6, 7, 8, 9], [3, 4, 5]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 5], [6, 7, 8, 9]))

        indices = Sampling.crossValidation(4, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([2, 3, 4, 5, 6, 7, 8, 9], [0, 1]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 5, 6, 7, 8, 9], [2, 3, 4]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 7, 8, 9], [5, 6]))
        self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 3, 4, 5, 6], [7, 8, 9]))

        indices = Sampling.crossValidation(numExamples, numExamples)
        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([1, 2, 3, 4, 5, 6, 7, 8, 9], [0]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 2, 3, 4, 5, 6, 7, 8, 9], [1]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 3, 4, 5, 6, 7, 8, 9], [2]))
        self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 4, 5, 6, 7, 8, 9], [3]))
        self.assertEquals((list(indices[4][0]), list(indices[4][1])), ([0, 1, 2, 3, 5, 6, 7, 8, 9], [4]))

        self.assertRaises(ValueError, Sampling.crossValidation, numExamples+1, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, 0, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, -1, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, folds, 1)

    def testBootstrap(self):
        numExamples = 10
        folds = 2

        indices = Sampling.bootstrap(folds, numExamples)

        for i in range(folds):
            self.assertEquals(indices[i][0].shape[0], numExamples)
            self.assertTrue(indices[i][1].shape[0] < numExamples)
            self.assertTrue((numpy.union1d(indices[0][0], indices[0][1]) == numpy.arange(numExamples)).all())

    def testBootstrap2(self):
        numExamples = 10
        folds = 2

        indices = Sampling.bootstrap2(folds, numExamples)

        for i in range(folds):
            self.assertEquals(indices[i][0].shape[0], numExamples)
            self.assertTrue(indices[i][1].shape[0] < numExamples)
            self.assertTrue((numpy.union1d(indices[0][0], indices[0][1]) == numpy.arange(numExamples)).all())

    def testShuffleSplit(self):
        numExamples = 10
        folds = 5

        indices = Sampling.shuffleSplit(folds, numExamples)

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())

        indices = Sampling.shuffleSplit(folds, numExamples, 0.5)
        trainSize = numExamples*0.5

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())
            self.assertTrue(indices[i][0].shape[0] == trainSize)

        indices = Sampling.shuffleSplit(folds, numExamples, 0.55)
        #print(indices)

if __name__ == '__main__':
    unittest.main()

<commit_msg>Test repeated cross validation. <commit_after>
import unittest
import numpy
from apgl.util.Sampling import Sampling

class  SamplingTest(unittest.TestCase):
    def testCrossValidation(self):
        numExamples = 10
        folds = 2

        indices = Sampling.crossValidation(folds, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([5, 6, 7, 8, 9], [0, 1, 2, 3, 4]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 3, 4], [5, 6, 7, 8, 9]))

        indices = Sampling.crossValidation(3, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([3, 4, 5, 6, 7, 8, 9], [0, 1, 2]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 2, 6, 7, 8, 9], [3, 4, 5]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 5], [6, 7, 8, 9]))

        indices = Sampling.crossValidation(4, numExamples)

        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([2, 3, 4, 5, 6, 7, 8, 9], [0, 1]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 1, 5, 6, 7, 8, 9], [2, 3, 4]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 2, 3, 4, 7, 8, 9], [5, 6]))
        self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 3, 4, 5, 6], [7, 8, 9]))

        indices = Sampling.crossValidation(numExamples, numExamples)
        self.assertEquals((list(indices[0][0]), list(indices[0][1])), ([1, 2, 3, 4, 5, 6, 7, 8, 9], [0]))
        self.assertEquals((list(indices[1][0]), list(indices[1][1])), ([0, 2, 3, 4, 5, 6, 7, 8, 9], [1]))
        self.assertEquals((list(indices[2][0]), list(indices[2][1])), ([0, 1, 3, 4, 5, 6, 7, 8, 9], [2]))
        self.assertEquals((list(indices[3][0]), list(indices[3][1])), ([0, 1, 2, 4, 5, 6, 7, 8, 9], [3]))
        self.assertEquals((list(indices[4][0]), list(indices[4][1])), ([0, 1, 2, 3, 5, 6, 7, 8, 9], [4]))

        self.assertRaises(ValueError, Sampling.crossValidation, numExamples+1, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, 0, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, -1, numExamples)
        self.assertRaises(ValueError, Sampling.crossValidation, folds, 1)

    def testBootstrap(self):
        numExamples = 10
        folds = 2

        indices = Sampling.bootstrap(folds, numExamples)

        for i in range(folds):
            self.assertEquals(indices[i][0].shape[0], numExamples)
            self.assertTrue(indices[i][1].shape[0] < numExamples)
            self.assertTrue((numpy.union1d(indices[0][0], indices[0][1]) == numpy.arange(numExamples)).all())

    def testBootstrap2(self):
        numExamples = 10
        folds = 2

        indices = Sampling.bootstrap2(folds, numExamples)

        for i in range(folds):
            self.assertEquals(indices[i][0].shape[0], numExamples)
            self.assertTrue(indices[i][1].shape[0] < numExamples)
            self.assertTrue((numpy.union1d(indices[0][0], indices[0][1]) == numpy.arange(numExamples)).all())

    def testShuffleSplit(self):
        numExamples = 10
        folds = 5

        indices = Sampling.shuffleSplit(folds, numExamples)

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())

        indices = Sampling.shuffleSplit(folds, numExamples, 0.5)
        trainSize = numExamples*0.5

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())
            self.assertTrue(indices[i][0].shape[0] == trainSize)

        indices = Sampling.shuffleSplit(folds, numExamples, 0.55)

    def testRepCrossValidation(self):
        numExamples = 10
        folds = 3
        repetitions = 1

        indices = Sampling.repCrossValidation(folds, numExamples, repetitions)

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())

        repetitions = 2
        indices = Sampling.repCrossValidation(folds, numExamples, repetitions)

        for i in range(folds):
            self.assertTrue((numpy.union1d(indices[i][0], indices[i][1]) == numpy.arange(numExamples)).all())

if __name__ == '__main__':
    unittest.main()

<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import os

from django.conf import settings

from libs.paths import delete_path, create_path

def get_experiment_outputs_path(experiment_name):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')
    return os.path.join(settings.OUTPUTS_ROOT, '/'.join(values))

def get_experiment_logs_path(experiment_name):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')
    return os.path.join(settings.LOGS_ROOT, '/'.join(values))

def delete_experiment_logs(experiment_group_name):
    path = get_experiment_logs_path(experiment_group_name)
    delete_path(path)

def delete_experiment_outputs(experiment_group_name):
    path = get_experiment_outputs_path(experiment_group_name)
    delete_path(path)

def create_experiment_logs_path(experiment_name):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')

    path = settings.LOGS_ROOT
    for value in values[:-1]:
        path = os.path.join(path, value)
        if not os.path.isdir(path):
            create_path(path)

    # Create file with permissions
    path = os.path.join(path, values[-1])
    open(path, 'w+')
    os.chmod(path, 0o777)

<commit_msg>Update experiments utils<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, division, print_function

import os

from django.conf import settings

from libs.paths import delete_path, create_path

def get_experiment_outputs_path(experiment_name):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')
    return os.path.join(settings.OUTPUTS_ROOT, '/'.join(values))

def get_experiment_logs_path(experiment_name):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')
    return os.path.join(settings.LOGS_ROOT, '/'.join(values))

def delete_experiment_logs(experiment_group_name):
    path = get_experiment_logs_path(experiment_group_name)
    delete_path(path)

def delete_experiment_outputs(experiment_group_name):
    path = get_experiment_outputs_path(experiment_group_name)
    delete_path(path)

def create_experiment_path(experiment_name, path):
    values = experiment_name.split('.')
    if len(values) == 3:
        values.insert(2, 'independents')

    for value in values[:-1]:
        path = os.path.join(path, value)
        if not os.path.isdir(path):
            create_path(path)

    return path

def create_experiment_logs_path(experiment_name):
    values = experiment_name.split('.')
    path = create_experiment_path(experiment_name, settings.LOGS_ROOT)

    # Create file with permissions
    path = os.path.join(path, values[-1])
    open(path, 'w+')
    os.chmod(path, 0o777)

def create_experiment_outputs_path(experiment_name):
    create_experiment_path(experiment_name, settings.OUTPUTS_ROOT)
<|endoftext|>


<commit_before>from django.db.models.signals import post_save
from django.dispatch import receiver

from libs.decorators import ignore_raw
from pipelines.constants import OperationStatuses
from pipelines.models import PipelineRunStatus, OperationRunStatus
from pipelines.tasks import (
    check_pipeline_run_status,
    start_operation_run,
    stop_pipeline_operation_runs,
    skip_pipeline_operation_runs,
)

@receiver(post_save, sender=PipelineRunStatus, dispatch_uid="new_pipeline_run_status_saved")
@ignore_raw
def new_pipeline_run_status(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if created:
        pipeline_run = instance.pipeline_run
        # Update job last_status
        pipeline_run.status = instance
        pipeline_run.save()
        # Notify operations with status change. This is necessary if we skip or stop the dag run.
        if pipeline_run.stopped():
            stop_pipeline_operation_runs.delay(pipeline_run_id=pipeline_run.id,
                                               message='Pipeline run was stopped')
        if pipeline_run.skipped():
            skip_pipeline_operation_runs.delay(pipeline_run_id=pipeline_run.id,
                                               message='Pipeline run was skipped')

@receiver(post_save, sender=OperationRunStatus, dispatch_uid="new_pipeline_run_status_saved")
@ignore_raw
def new_operation_run_status(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if not created:
        return

    operation_run = instance.operation_run
    pipeline_run = operation_run.pipeline_run
    # Update job last_status
    operation_run.status = instance
    operation_run.save()

    # Check if we need to update the pipeline_run's status
    check_pipeline_run_status.delay(pipline_run_uuid=pipeline_run.uuid.hex,
                                    status=instance.status,
                                    message=instance.message)
    if operation_run.is_done():
        # Notify downstream that instance is done, and that its dependency can start.
        for op_run in operation_run.downstream_runs.filter(status=OperationStatuses.CREATED):
            start_operation_run.delay(operation_run_id=op_run.id)
<commit_msg>Add pipeline and operation run signals to handle created status<commit_after>from django.db.models.signals import post_save
from django.dispatch import receiver

from libs.decorators import ignore_raw
from pipelines.constants import OperationStatuses, PipelineStatuses
from pipelines.models import PipelineRunStatus, OperationRunStatus, PipelineRun, OperationRun
from pipelines.tasks import (
    check_pipeline_run_status,
    start_operation_run,
    stop_pipeline_operation_runs,
    skip_pipeline_operation_runs,
)

@receiver(post_save, sender=PipelineRun, dispatch_uid="pipeline_run_saved")
@ignore_raw
def new_pipeline_run(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if not created:
        return

    instance.set_status(PipelineStatuses.CREATED)

@receiver(post_save, sender=OperationRun, dispatch_uid="operation_run_saved")
@ignore_raw
def new_operation_run(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if not created:
        return

    instance.set_status(OperationStatuses.CREATED)

@receiver(post_save, sender=PipelineRunStatus, dispatch_uid="new_pipeline_run_status_saved")
@ignore_raw
def new_pipeline_run_status(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if created:
        pipeline_run = instance.pipeline_run
        # Update job last_status
        pipeline_run.status = instance
        pipeline_run.save()
        # Notify operations with status change. This is necessary if we skip or stop the dag run.
        if pipeline_run.stopped():
            stop_pipeline_operation_runs.delay(pipeline_run_id=pipeline_run.id,
                                               message='Pipeline run was stopped')
        if pipeline_run.skipped():
            skip_pipeline_operation_runs.delay(pipeline_run_id=pipeline_run.id,
                                               message='Pipeline run was skipped')

@receiver(post_save, sender=OperationRunStatus, dispatch_uid="new_pipeline_run_status_saved")
@ignore_raw
def new_operation_run_status(sender, **kwargs):
    instance = kwargs['instance']
    created = kwargs.get('created', False)

    if not created:
        return

    operation_run = instance.operation_run
    pipeline_run = operation_run.pipeline_run
    # Update job last_status
    operation_run.status = instance
    operation_run.save()

    # No need to check if it is just created
    if instance.status == OperationStatuses.CREATED:
        return

    # Check if we need to update the pipeline_run's status
    check_pipeline_run_status.delay(pipline_run_uuid=pipeline_run.uuid.hex,
                                    status=instance.status,
                                    message=instance.message)
    if operation_run.is_done():
        # Notify downstream that instance is done, and that its dependency can start.
        for op_run in operation_run.downstream_runs.filter(status=OperationStatuses.CREATED):
            start_operation_run.delay(operation_run_id=op_run.id)
<|endoftext|>


<commit_before># -*- coding: cp1251 -*-

from pony.main import *

use_autoreload()

@http
def page1():
    return 'Hello, world!'

@http(type='text/html')
def page2():
    return """

This is HTML page

Click here for return
           """

@http
def page3():
    "This page deliberately contains error"
    a = u''
    b = None   #
    c = a / b  #
    return a

@http
@printhtml
def page4():
    "Using of @printhtml decorator"
    print '<title>Page 3</title>'
    print '
Hello again!
'
    print '
'
    print '
Demonstration of @printhtml decorator'
    print '
Click here for return'

@printhtml
def header(title='Demonstration of Pony features'):
    print '<title>%s</title>' % title
    print '
%s
' % title
    print '
'

@printhtml
def footer():
    print '
Click here to return to main page'

@webpage('mypage.html')
def page5():
    "Page with custom URL"
    print header('Demonstration of custom URL')
    print '
This page have custom URL'
    print '
It also demonstrate how common parts of page'
    print '(such as header or footer) can be factored out'
    print footer()

@webpage('/myblog/archives/$year/posts.html')
@webpage('/myblog/archives/$year/$month/posts.html')
@webpage('/myblog/$lang/archives/$year/posts.html')
@webpage('/myblog/$lang/archives/$year/$month/posts.html')
def page6(year, month=None, lang='en'):
    "Parameters encoded in URL"
    print header('My Blog Archives')
    print '
Demonstration how parameters can be encoded in URL'
    print '
'
    print '
Language: %s' % lang
    print '
Year: %s' % year
    print '
Month: %s' % (month or 'Not given')
    print '
'
    print '
Go to year 2003
' % url(page6, '2003')
    print ('
Go to French 2005-11
' % url(page6, 2005, 11, 'fr'))
    print '
%s
' % link('Go to English 2004-10', page6, 2004, 10)
    print footer()

@webpage('/hello?first_name=$name')
def page7(name=None):
    "Parameters in query part of URL"
    print header('URL parameters')
    if name is None:
        print '''

What is your name?

              <form>
              <input type="text" name="first_name">
              <input type="submit" value="Send!">
              </form>

You can try input something "illegal"
              instead of name, such as

<script>alert("You are hacked!");</script>

You will see as Pony automatically prevent
              such XSS (Cross-Site Scripting) attacks
              (those script will not be executed)

              '''
    else:
        print '
Hello, %s!
' % name
        print '
Try again
' % url(page7)
    print footer()

@webpage
def page8():
    "Using of html() function"
    return html()

@webpage
def page9():
    "Tabs"
    return html("""
    <link jquery plugins="tabs">

Example of tabs

    $tabs()
    $.tab("One") {

Nested tabs:

            $tabs(class_="span-12 prepend-1 append-1 last")
            $.tab("Nested tab 1"){
Nested tab ONE
}
            $.tab("Nested tab 2"){
Nested tab TWO
}
            $.tab("Tab with very very long name"){
Nested tab THREE
}
    }
    $.tab("Two") {

Content of second tab

    }
    $.tab("Three") {

Tab three

One

Two

Three

    }

Return to main page
    """)

@webpage('/') # This is root page
def index():
    print header('Simple Pony examples')
    print '
'
    print '

HelloWorld example

' % url(page1)
    print '

%s

' % link('Simplest HTML page', page2)
    print '

%s

' % link(page3)
    print '

%s

' % link(page4)
    print '

%s

' % link(page5)
    print '

%s

' % link(page6, 2007, 10)
    print '

%s

' % link(page7)
    print '

%s

' % link(page8)
    print '

%s

' % link(page9)
    print '
'
    print '

automatically obfuscated e-mail
'
    print '
External link
'
    print '
FTP link
'
    print '''
JavaScript url
'''

if __name__ == '__main__':
    http.start()
    # show_gui()
<commit_msg>Bug in example (invalid markup)<commit_after># -*- coding: cp1251 -*-

from pony.main import *

use_autoreload()

@http
def page1():
    return 'Hello, world!'

@http(type='text/html')
def page2():
    return """

This is HTML page

Click here for return
           """

@http
def page3():
    "This page deliberately contains error"
    a = u''
    b = None   #
    c = a / b  #
    return a

@http
@printhtml
def page4():
    "Using of @printhtml decorator"
    print '<title>Page 3</title>'
    print '
Hello again!
'
    print '
'
    print '
Demonstration of @printhtml decorator'
    print '
Click here for return'

@printhtml
def header(title='Demonstration of Pony features'):
    print '<title>%s</title>' % title
    print '
%s
' % title
    print '
'

@printhtml
def footer():
    print '
Click here to return to main page'

@webpage('mypage.html')
def page5():
    "Page with custom URL"
    print header('Demonstration of custom URL')
    print '
This page have custom URL'
    print '
It also demonstrate how common parts of page'
    print '(such as header or footer) can be factored out'
    print footer()

@webpage('/myblog/archives/$year/posts.html')
@webpage('/myblog/archives/$year/$month/posts.html')
@webpage('/myblog/$lang/archives/$year/posts.html')
@webpage('/myblog/$lang/archives/$year/$month/posts.html')
def page6(year, month=None, lang='en'):
    "Parameters encoded in URL"
    print header('My Blog Archives')
    print '
Demonstration how parameters can be encoded in URL'
    print '
'
    print '
Language: %s' % lang
    print '
Year: %s' % year
    print '
Month: %s' % (month or 'Not given')
    print '
'
    print '
Go to year 2003
' % url(page6, '2003')
    print ('
Go to French 2005-11
' % url(page6, 2005, 11, 'fr'))
    print '
%s
' % link('Go to English 2004-10', page6, 2004, 10)
    print footer()

@webpage('/hello?first_name=$name')
def page7(name=None):
    "Parameters in query part of URL"
    print header('URL parameters')
    if name is None:
        print '''

What is your name?

              <form>
              <input type="text" name="first_name">
              <input type="submit" value="Send!">
              </form>

You can try input something "illegal"
              instead of name, such as

<script>alert("You are hacked!");</script>

You will see as Pony automatically prevent
              such XSS (Cross-Site Scripting) attacks
              (those script will not be executed)

              '''
    else:
        print '
Hello, %s!
' % name
        print '
Try again
' % url(page7)
    print footer()

@webpage
def page8():
    "Using of html() function"
    return html()

@webpage
def page9():
    "Tabs"
    return html("""
    <link jquery plugins="tabs">

Example of tabs

    $tabs()
    $.tab("One") {

Nested tabs:

            $tabs(class_="span-12 prepend-1 append-1 last")
            $.tab("Nested tab 1"){
Nested tab ONE
}
            $.tab("Nested tab 2"){
Nested tab TWO
}
            $.tab("Tab with very very long name"){
Nested tab THREE
}
    }
    $.tab("Two") {

Content of second tab

    }
    $.tab("Three") {

Tab three

One

Two

Three

    }

Return to main page
    """)

@webpage('/') # This is root page
def index():
    print header('Simple Pony examples')
    print '
'
    print '

HelloWorld example

' % url(page1)
    print '

%s

' % link('Simplest HTML page', page2)
    print '

%s

' % link(page3)
    print '

%s

' % link(page4)
    print '

%s

' % link(page5)
    print '

%s

' % link(page6, 2007, 10)
    print '

%s

' % link(page7)
    print '

%s

' % link(page8)
    print '

%s

' % link(page9)
    print '
'
    print '

automatically obfuscated e-mail
'
    print '
External link
'
    print '
FTP link
'
    print '''
JavaScript url
'''

if __name__ == '__main__':
    http.start()
    # show_gui()
<|endoftext|>


<commit_before>import random

# DEFINITIONS
# note base fitness is 1
# k-mutant fitness is 1 + mutant_traits[k][0]
# form of mutant_traits is [... (w_k, mu_k) ...]
# population is dict of k:N_k mutant categry to mutant pop

N = 100  # total fixed pop
mutant_traits = [(0.0, 0.1),   # base pop
                 (-0.1, 0.1),  # 1-mutant
                 (-0.2, 0.1),  # 2-mutant etc
                 (0.05, 0.0)]
mutant_fitness = [1 + pair[0] for pair in mutant_traits]
K = len(mutant_traits) - 1
print " simulating a %d-hit process on %d individuals" % (K, N)

# ensure mutation fitness deviations are well-defined
assert(1.0/float(N) < mutant_traits[-1][0] < 1.0)  # K-mutant moderately fit
assert(mutant_traits[-1][1] == 0.0)  # K-mutants don't mutate
for i in xrange(K):
    assert(mutant_traits[i][0]) <= 0  # intermediates less fit
    assert(mutant_traits[i][1]) > 0  # intermediates can mutate

def count_pop(pop_dict):
    return sum(pop_dict.values())

# INITIALIZE
population[0] = N  # note initial pop assumed to have 0 mutants

# SIMULATE
# 0. set timestep dt (they use dt = 10^-2 = 0.01 generations)
# 1. calculate mean fitness w_bar
# 2. each k-mutant does either A, B, or C independently
#        A: divides into two k_mutants, prob (1 + w_k - w_bar)*dt
#        B: divides with one k-mutant, one k+1 mutant, prob (mu_k)*dt
#        C: dies, prob (1)*dt
#        note there are thus 3N indep events per timestep
# 3. if pop N* after 2. is not N, then multiply each N_k by N/N* and round to nearest int
# 4. repeat until N_K = N (i.e. whole pop is K-mutants)
# 5. at end of run, return last time there were no K-mutants, call this
#    "time to production of first successful K-mutant"

#number_to_mutate
#random.sample(range[N], number_to_mutate)
<commit_msg>it runs<commit_after>import numpy as np
import random

# DEFINITIONS
# note base fitness is 1
# k-mutant fitness is 1 + mutant_traits[k][0]
# form of mutant_traits is [... (w_k, mu_k) ...]
# population is dict of k:N_k mutant categry to mutant pop

N = 100  # total fixed pop
mutant_traits = [(0.0, 0.1),   # base pop
                 (-0.1, 0.1),  # 1-mutant
                 (-0.2, 0.1),  # 2-mutant etc
                 (0.05, 0.0)]
mutant_fitness = [1 + pair[0] for pair in mutant_traits]
K = len(mutant_traits) - 1
population = {k:0 for k in xrange(K+1)}
print " simulating a %d-hit process on %d individuals" % (K, N)

# ensure mutation fitness deviations are well-defined
assert(1.0/float(N) < mutant_traits[-1][0] < 1.0)  # K-mutant moderately fit
assert(mutant_traits[-1][1] == 0.0)  # K-mutants don't mutate
for i in xrange(K):
    assert(mutant_traits[i][0]) <= 0  # intermediates less fit
    assert(mutant_traits[i][1]) > 0  # intermediates can mutate

# FUNCTIONS

def count_pop(pop_dict):
    return sum(pop_dict.values())

def get_mean_fitness(pop_dict):
    mean_fitness = sum(mutant_fitness[i]*pop_dict[i] for i in xrange(K+1)) / N
    return mean_fitness

def sample_probabilities(k, dt, mean_fitness):
    p_a = dt*(1 + mutant_fitness[k] - mean_fitness)
    p_b = dt*mutant_traits[k][1]
    p_c = dt
    rgn = np.random.rand(3)
    return rgn[0] < p_a, rgn[1] < p_b, rgn[2] < p_c

def increment_pop(increment_list, pop_dict):
    for k in xrange(K+1):
        pop_dict[k] += increment_list[k]
    if count_pop(pop_dict) != N:
        return normalize_pop(pop_dict)
    else:
        return pop_dict

def normalize_pop(pop_dict):
    weight = float(N) / count_pop(pop_dict)
    for k in xrange(K+1):
        pop_dict[k] = int(round(pop_dict[k]*weight))
    return pop_dict

# INITIALIZE
population[0] = N  # note initial pop assumed to have 0 mutants
print get_mean_fitness(population)
print count_pop(population)

# SIMULATE (see p9, Section 5 of paper)
dt = 0.01
t = 0.0
# 0. set timestep dt (they use dt = 10^-2 = 0.01 generations)
# 1. calculate mean fitness w_bar
# 2. each k-mutant does either A, B, or C independently
#        A: divides into two k_mutants, prob (1 + w_k - w_bar)*dt
#        B: divides with one k-mutant, one k+1 mutant, prob (mu_k)*dt
#        C: dies, prob (1)*dt
#        note there are thus 3N indep events per timestep
# 3. if pop N* after 2. is not N, then multiply each N_k by N/N* and round to nearest int
# 4. repeat until N_K = N (i.e. whole pop is K-mutants)
# 5. at end of run, return last time there were no K-mutants, call this
#    "time to production of first successful K-mutant"
while 1:
    t += dt
    w_bar = get_mean_fitness(population)
    increments = [0 for k in xrange(K+1)]
    for k in xrange(K+1):
        for n in xrange(population[k]):
            (A, B, C) = sample_probabilities(k, dt, w_bar)  # TODO fix this part maybe wrong
            if A:
                increments[k] += 1
            if B:
                increments[k] -= 1
                increments[k+1] += 1
            if C:
                increments[k] -= 1
    population = increment_pop(increments, population)
    print population
    if population[K] == N:
        print "K-mutants have fixated at time %.2f (%d steps)" % (t, t/dt)
        break

print "broke from loop"

#number_to_mutate
#random.sample(range[N], number_to_mutate)
<|endoftext|>


<commit_before>import numpy as np
import random

# DEFINITIONS
# note base fitness is 1
# k-mutant fitness is 1 + mutant_traits[k][0]
# form of mutant_traits is [... (w_k, mu_k) ...]
# population is dict of k:N_k mutant categry to mutant pop

N = 100  # total fixed pop
mutant_traits = [(0.0, 0.1),   # base pop
                 (-0.1, 0.1),  # 1-mutant
                 (-0.2, 0.1),  # 2-mutant etc
                 (0.05, 0.0)]
mutant_fitness = [1 + pair[0] for pair in mutant_traits]
K = len(mutant_traits) - 1
population = {k:0 for k in xrange(K+1)}
print " simulating a %d-hit process on %d individuals" % (K, N)

# ensure mutation fitness deviations are well-defined
assert(1.0/float(N) < mutant_traits[-1][0] < 1.0)  # K-mutant moderately fit
assert(mutant_traits[-1][1] == 0.0)  # K-mutants don't mutate
for i in xrange(K):
    assert(mutant_traits[i][0]) <= 0  # intermediates less fit
    assert(mutant_traits[i][1]) > 0  # intermediates can mutate

# FUNCTIONS

def count_pop(pop_dict):
    return sum(pop_dict.values())

def get_mean_fitness(pop_dict):
    mean_fitness = sum(mutant_fitness[i]*pop_dict[i] for i in xrange(K+1)) / N
    return mean_fitness

def sample_probabilities(k, dt, mean_fitness):
    p_a = dt*(1 + mutant_fitness[k] - mean_fitness)
    p_b = dt*mutant_traits[k][1]
    p_c = dt
    rgn = np.random.rand(3)
    return rgn[0] < p_a, rgn[1] < p_b, rgn[2] < p_c

def increment_pop(increment_list, pop_dict):
    for k in xrange(K+1):
        pop_dict[k] += increment_list[k]
    if count_pop(pop_dict) != N:
        return normalize_pop(pop_dict)
    else:
        return pop_dict

def normalize_pop(pop_dict):
    weight = float(N) / count_pop(pop_dict)
    for k in xrange(K+1):
        pop_dict[k] = int(round(pop_dict[k]*weight))
    return pop_dict

# INITIALIZE
population[0] = N  # note initial pop assumed to have 0 mutants
print get_mean_fitness(population)
print count_pop(population)

# SIMULATE (see p9, Section 5 of paper)
dt = 0.01
t = 0.0
# 0. set timestep dt (they use dt = 10^-2 = 0.01 generations)
# 1. calculate mean fitness w_bar
# 2. each k-mutant does either A, B, or C independently
#        A: divides into two k_mutants, prob (1 + w_k - w_bar)*dt
#        B: divides with one k-mutant, one k+1 mutant, prob (mu_k)*dt
#        C: dies, prob (1)*dt
#        note there are thus 3N indep events per timestep
# 3. if pop N* after 2. is not N, then multiply each N_k by N/N* and round to nearest int
# 4. repeat until N_K = N (i.e. whole pop is K-mutants)
# 5. at end of run, return last time there were no K-mutants, call this
#    "time to production of first successful K-mutant"
while 1:
    t += dt
    w_bar = get_mean_fitness(population)
    increments = [0 for k in xrange(K+1)]
    for k in xrange(K+1):
        for n in xrange(population[k]):
            (A, B, C) = sample_probabilities(k, dt, w_bar)  # TODO fix this part maybe wrong
            if A:
                increments[k] += 1
            if B:
                increments[k] -= 1
                increments[k+1] += 1
            if C:
                increments[k] -= 1
    population = increment_pop(increments, population)
    print population
    if population[K] == N:
        print "K-mutants have fixated at time %.2f (%d steps)" % (t, t/dt)
        break

print "broke from loop"

#number_to_mutate
#random.sample(range[N], number_to_mutate)
<commit_msg>bracket adjust<commit_after>import numpy as np
import random

# DEFINITIONS
# note base fitness is 1
# k-mutant fitness is 1 + mutant_traits[k][0]
# form of mutant_traits is [... (w_k, mu_k) ...]
# population is dict of k:N_k mutant categry to mutant pop

N = 100  # total fixed pop
mutant_traits = [(0.0, 0.1),   # base pop
                 (-0.1, 0.1),  # 1-mutant
                 (-0.2, 0.1),  # 2-mutant etc
                 (0.05, 0.0)]
mutant_fitness = [1 + pair[0] for pair in mutant_traits]
K = len(mutant_traits) - 1
population = {k:0 for k in xrange(K+1)}
print " simulating a %d-hit process on %d individuals" % (K, N)

# ensure mutation fitness deviations are well-defined
assert(1.0/float(N) < mutant_traits[-1][0] < 1.0)  # K-mutant moderately fit
assert(mutant_traits[-1][1] == 0.0)  # K-mutants don't mutate
for i in xrange(K):
    assert(mutant_traits[i][0] <= 0)  # intermediates less fit
    assert(mutant_traits[i][1] > 0)  # intermediates can mutate

# FUNCTIONS

def count_pop(pop_dict):
    return sum(pop_dict.values())

def get_mean_fitness(pop_dict):
    mean_fitness = sum(mutant_fitness[i]*pop_dict[i] for i in xrange(K+1)) / N
    return mean_fitness

def sample_probabilities(k, dt, mean_fitness):
    p_a = dt*(1 + mutant_fitness[k] - mean_fitness)
    p_b = dt*mutant_traits[k][1]
    p_c = dt
    rgn = np.random.rand(3)
    return rgn[0] < p_a, rgn[1] < p_b, rgn[2] < p_c

def increment_pop(increment_list, pop_dict):
    for k in xrange(K+1):
        pop_dict[k] += increment_list[k]
    if count_pop(pop_dict) != N:
        return normalize_pop(pop_dict)
    else:
        return pop_dict

def normalize_pop(pop_dict):
    weight = float(N) / count_pop(pop_dict)
    for k in xrange(K+1):
        pop_dict[k] = int(round(pop_dict[k]*weight))
    return pop_dict

# INITIALIZE
population[0] = N  # note initial pop assumed to have 0 mutants
print get_mean_fitness(population)
print count_pop(population)

# SIMULATE (see p9, Section 5 of paper)
dt = 0.01
t = 0.0
# 0. set timestep dt (they use dt = 10^-2 = 0.01 generations)
# 1. calculate mean fitness w_bar
# 2. each k-mutant does either A, B, or C independently
#        A: divides into two k_mutants, prob (1 + w_k - w_bar)*dt
#        B: divides with one k-mutant, one k+1 mutant, prob (mu_k)*dt
#        C: dies, prob (1)*dt
#        note there are thus 3N indep events per timestep
# 3. if pop N* after 2. is not N, then multiply each N_k by N/N* and round to nearest int
# 4. repeat until N_K = N (i.e. whole pop is K-mutants)
# 5. at end of run, return last time there were no K-mutants, call this
#    "time to production of first successful K-mutant"
while 1:
    t += dt
    w_bar = get_mean_fitness(population)
    increments = [0 for k in xrange(K+1)]
    for k in xrange(K+1):
        for n in xrange(population[k]):
            (A, B, C) = sample_probabilities(k, dt, w_bar)  # TODO fix this part maybe wrong
            if A:
                increments[k] += 1
            if B:
                increments[k] -= 1
                increments[k+1] += 1
            if C:
                increments[k] -= 1
    population = increment_pop(increments, population)
    print population
    if population[K] == N:
        print "K-mutants have fixated at time %.2f (%d steps)" % (t, t/dt)
        break

print "broke from loop"

#number_to_mutate
#random.sample(range[N], number_to_mutate)
<|endoftext|>


<commit_before>#!/usr/bin/python
# -*- encoding: utf8 -*-

# Based on: http://pyod.googlecode.com/hg/Gzipper.py
# Modified to use six.StringIO instead of cStringIO.StringIO()
# Latest changes: 24 Sep 2015

'''
    A WSGI middleware application that automatically gzips output
    to the client.
    Before doing any gzipping, it first checks the environ to see if
    the client can even support gzipped output. If not, it immediately
    drops out.
    It automatically modifies the headers to include the proper values
    for the 'Accept-Encoding' and 'Vary' headers.

    Example of use:

        from wsgiref.simple_server import WSGIServer, WSGIRequestHandler

        def test_app(environ, start_response):
            status = '200 OK'
            headers = [('content-type', 'text/html')]
            start_response(status, headers)
            return ['Hello gzipped world!']

        app = Gzipper(test_app, compresslevel=8)
        httpd = WSGIServer(('', 8080), WSGIRequestHandler)
        httpd.set_app(app)
        httpd.serve_forever()
'''

from gzip import GzipFile
import six

__version__ = '1.0.1'

def compress(data, compression_level):
    ''' The `gzip` module didn't provide a way to gzip just a string.
        Had to hack together this. I know, it isn't pretty.
    '''
    buf = six.StringIO()
    gz_file = GzipFile(None, 'wb', compression_level, buf)
    gz_file.write(data)
    gz_file.close()
    return buf.getvalue()

def parse_encoding_header(header):
    ''' Break up the `HTTP_ACCEPT_ENCODING` header into a dict of
        the form, {'encoding-name':qvalue}.
    '''
    encodings = {'identity': 1.0}
    for encoding in header.split(','):
        if encoding.find(';') > -1:
            encoding, qvalue = encoding.split(';')
            encoding = encoding.strip()
            qvalue = qvalue.split('=', 1)[1]
            if qvalue != '':
                encodings[encoding] = float(qvalue)
            else:
                encodings[encoding] = 1
        else:
            encodings[encoding] = 1
    return encodings

def client_wants_gzip(accept_encoding_header):
    ''' Check to see if the client can accept gzipped output, and whether
        or not it is even the preferred method. If `identity` is higher, then
        no gzipping should occur.
    '''
    encodings = parse_encoding_header(accept_encoding_header)
    if 'gzip' in encodings:
        return encodings['gzip'] >= encodings['identity']
    elif '*' in encodings:
        return encodings['*'] >= encodings['identity']
    else:
        return False

DEFAULT_COMPRESSABLES = set([
    'text/plain', 'text/html', 'text/css',
    'application/json', 'application/x-javascript', 'text/xml',
    'application/xml', 'application/xml+rss', 'text/javascript'
])

class Gzipper(object):
    ''' WSGI middleware to wrap around and gzip all output.
        This automatically adds the content-encoding header.
    '''
    def __init__(self, app, content_types=DEFAULT_COMPRESSABLES,
                 compresslevel=6):
        self.app = app
        self.content_types = content_types
        self.compresslevel = compresslevel

    def __call__(self, environ, start_response):
        ''' Do the actual work. If the host doesn't support gzip
            as a proper encoding,then simply pass over to the
            next app on the wsgi stack.
        '''
        if not client_wants_gzip(environ.get('HTTP_ACCEPT_ENCODING', '')):
            return self.app(environ, start_response)

        buf = {'to_gzip': False, 'body': ''}

        def _write(body):
            # for WSGI compliance
            buf['body'] = body

        def _start_response(status, headers, exc_info=None):
            ''' Wrapper around the original `start_response` function.
                The sole purpose being to add the proper headers automatically.
            '''
            for header in headers:
                field = header[0].lower()
                if field == 'content-encoding':
                    # if the content is already encoded, don't compress
                    buf['to_gzip'] = False
                    break
                elif field == 'content-type':
                    ctype = header[1].split(';')[0]
                    if ctype in self.content_types and not(
                            'msie' in environ.get(
                                'HTTP_USER_AGENT', '').lower()
                            and 'javascript' in ctype):
                        buf['to_gzip'] = True

            buf['status'] = status
            buf['headers'] = headers
            buf['exc_info'] = exc_info
            return _write

        data = self.app(environ, _start_response)

        if buf['status'].startswith('200 ') and buf['to_gzip']:
            data = ''.join(data)
            if len(data) > 200:
                data = compress(data, self.compresslevel)
                headers = buf['headers']
                headers.append(('Content-Encoding', 'gzip'))
                headers.append(('Vary', 'Accept-Encoding'))
                for i, header in enumerate(headers):
                    if header[0] == 'Content-Length':
                        headers[i] = (
                            'Content-Length', str(len(data) + len(buf['body']))
                        )
                        break
            data = [data]

        _writable = start_response(buf['status'], buf['headers'],
                                   buf['exc_info'])

        if buf['body']:
            _writable(buf['body'])

        return data
<commit_msg>v0.02, SQL search-anywhere fragments<commit_after><|endoftext|>


<commit_before>#
# Copyright 2017 National Renewable Energy Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
    This program executes OpenFAST on all of the CertTest cases. It mimics the
    regression test execution through CMake/CTest. All generated data goes into
    `openfast/build/reg_tests`.

    Usage: `python manualRegressionTest.py openfast/install/bin/openfast [Darwin,Linux,Windows] [Intel,GNU] tolerance`
"""

import os
import sys
basepath = os.path.sep.join(sys.argv[0].split(os.path.sep)[:-1]) if os.path.sep in sys.argv[0] else "."
sys.path.insert(0, os.path.sep.join([basepath, "lib"]))
import argparse
import subprocess
import rtestlib as rtl

def strFormat(string):
    return "{:<" + str(len(string)) + "}"

### Verify input arguments
parser = argparse.ArgumentParser(description="Executes OpenFAST and a regression test for a single test case.")
parser.add_argument("executable", metavar="OpenFAST", type=str, nargs=1, help="path to the OpenFAST executable")
parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="current system's name: [Darwin,Linux,Windows]")
parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="compiler's id: [Intel,GNU]")
parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="tolerance defining pass or failure in the regression test")
parser.add_argument("-p", "-plot", dest="plot", default=False, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include matplotlib plots in failed cases")
parser.add_argument("-n", "-no-exec", dest="noExec", default=False, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases")
parser.add_argument("-v", "-verbose", dest="verbose", default=False, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output")
parser.add_argument("-case", dest="case", default="", metavar="Case-Name", type=str, nargs="?", help="single case name to execute")

args = parser.parse_args()
openfast_executable = args.executable[0]
sourceDirectory = ".."
buildDirectory = os.path.join("..", "build", "reg_tests", "glue-codes", "fast")
machine = args.systemName[0]
compiler = args.compilerId[0]
tolerance = args.tolerance[0]
plotError = args.plot if args.plot is False else True
plotFlag = "-p" if plotError else ""
noExec = args.noExec if args.noExec is False else True
noExecFlag = "-n" if noExec else ""
verbose = args.verbose if args.verbose is False else True
case = args.case

outstd = sys.stdout if verbose else open(os.devnull, 'w')
pythonCommand = sys.executable

if case is not "":
    caselist = [case]
else:
    with open(os.path.join("r-test", "glue-codes", "fast", "CaseList.md")) as listfile:
        caselist = listfile.readlines()
casenames = [c.rstrip("\n\r").strip() for c in caselist if "#" not in c]

results = []
prefix, passString, failString = "executing", "[PASS]", "[FAIL]"
longestName = max(casenames, key=len)
for case in casenames:
    print(strFormat(prefix).format(prefix), strFormat(longestName).format(case), end="", flush=True)
    command = "{} executeOpenfastRegressionCase.py {} {} {} {} {} {} {} {} {}".format(pythonCommand, case, openfast_executable, sourceDirectory, buildDirectory, tolerance, machine, compiler, plotFlag, noExecFlag)
    returnCode = subprocess.call(command, stdout=outstd, shell=True)
    resultString = passString if returnCode == 0 else failString
    results.append((strFormat(longestName).format(case), resultString))
    print(resultString)

print("\nRegression test execution completed with these results:")
for r in results:
    print(" ".join([str(rr) for rr in r]))

nPasses = len( [r[1] for r in results if r[1] == passString] )
print("Total PASSING tests - {}".format(nPasses))
print("Total FAILING tests - {}".format(len(results) - nPasses))
<commit_msg>improve the results reporting<commit_after>#
# Copyright 2017 National Renewable Energy Laboratory
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

"""
    This program executes OpenFAST on all of the CertTest cases. It mimics the
    regression test execution through CMake/CTest. All generated data goes into
    `openfast/build/reg_tests`.

    Usage: `python manualRegressionTest.py openfast/install/bin/openfast [Darwin,Linux,Windows] [Intel,GNU] tolerance`
"""

import os
import sys
basepath = os.path.sep.join(sys.argv[0].split(os.path.sep)[:-1]) if os.path.sep in sys.argv[0] else "."
sys.path.insert(0, os.path.sep.join([basepath, "lib"]))
import argparse
import subprocess

def strFormat(string):
    return "{:<" + str(len(string)) + "}"

### Verify input arguments
parser = argparse.ArgumentParser(description="Executes OpenFAST and a regression test for a single test case.")
parser.add_argument("executable", metavar="OpenFAST", type=str, nargs=1, help="path to the OpenFAST executable")
parser.add_argument("systemName", metavar="System-Name", type=str, nargs=1, help="current system's name: [Darwin,Linux,Windows]")
parser.add_argument("compilerId", metavar="Compiler-Id", type=str, nargs=1, help="compiler's id: [Intel,GNU]")
parser.add_argument("tolerance", metavar="Test-Tolerance", type=float, nargs=1, help="tolerance defining pass or failure in the regression test")
parser.add_argument("-p", "-plot", dest="plot", default=False, metavar="Plotting-Flag", type=bool, nargs="?", help="bool to include matplotlib plots in failed cases")
parser.add_argument("-n", "-no-exec", dest="noExec", default=False, metavar="No-Execution", type=bool, nargs="?", help="bool to prevent execution of the test cases")
parser.add_argument("-v", "-verbose", dest="verbose", default=False, metavar="Verbose-Flag", type=bool, nargs="?", help="bool to include verbose system output")
parser.add_argument("-case", dest="case", default="", metavar="Case-Name", type=str, nargs="?", help="single case name to execute")

args = parser.parse_args()
openfast_executable = args.executable[0]
sourceDirectory = ".."
buildDirectory = os.path.join("..", "build", "reg_tests", "glue-codes", "fast")
machine = args.systemName[0]
compiler = args.compilerId[0]
tolerance = args.tolerance[0]
plotError = args.plot if args.plot is False else True
plotFlag = "-p" if plotError else ""
noExec = args.noExec if args.noExec is False else True
noExecFlag = "-n" if noExec else ""
verbose = args.verbose if args.verbose is False else True
case = args.case

outstd = sys.stdout if verbose else open(os.devnull, 'w')
pythonCommand = sys.executable

if case is not "":
    caselist = [case]
else:
    with open(os.path.join("r-test", "glue-codes", "fast", "CaseList.md")) as listfile:
        caselist = listfile.readlines()
casenames = [c.rstrip("\n\r").strip() for c in caselist if "#" not in c]

results = []
prefix, passString, failString = "executing", "PASS", "FAIL"
longestName = max(casenames, key=len)
for case in casenames:
    print(strFormat(prefix).format(prefix), strFormat(longestName+" ").format(case), end="", flush=True)
    command = "{} executeOpenfastRegressionCase.py {} {} {} {} {} {} {} {} {}".format(pythonCommand, case, openfast_executable, sourceDirectory, buildDirectory, tolerance, machine, compiler, plotFlag, noExecFlag)
    returnCode = subprocess.call(command, stdout=outstd, shell=True)
    resultString = passString if returnCode == 0 else failString
    results.append((case, resultString))
    print(resultString)

from errorPlotting import exportResultsSummary
exportResultsSummary(buildDirectory, results)

print("\nRegression test execution completed with these results:")
for r in results:
    print(" ".join([strFormat(longestName).format(r[0]), r[1]]))

nPasses = len( [r[1] for r in results if r[1] == passString] )
print("Total PASSING tests - {}".format(nPasses))
print("Total FAILING tests - {}".format(len(results) - nPasses))
<|endoftext|>


<commit_before>from sklearn.cross_validation import train_test_split
from sklearn.metrics import confusion_matrix
import numpy as np
import pandas as pd

def logistic_regression(clf, X, Y, train_size=0.8, random_state=1):
    # select random train and test set from X and Y
    X_train, X_test, t_train, t_test = train_test_split(X, Y,
                                                        train_size=train_size,
                                                        random_state=random_state)
    # fit the training data to the passed in model
    clf.fit(X_train, t_train)

    # create beta coeff dataframe with intercept if intercept=True
    preds = clf.predict(X_test)
    betas = clf.coef_[0]
    if clf.intercept_ > 0:
        betas = np.append(betas, clf.intercept_[0])

    colnames = X.columns.values
    if clf.intercept_ > 0:
        colnames = np.append(colnames, 'intercept')

    coef_table = pd.DataFrame({'coefficients': betas, '_variables': colnames})

    # calcualte confusion matrix common responses: sensitivity, specificity, precision
    conf_matrix = confusion_matrix(t_test, preds)
    true_pos = conf_matrix[0,0]
    true_neg = conf_matrix[1,1]
    false_pos = conf_matrix[0,1]
    false_neg = conf_matrix[1,0]

    sensitivity = round(true_pos / float((true_pos + false_neg)), 4)
    specificity = round(true_neg / float((false_pos + true_neg)), 4)
    precision = round(true_pos / float((true_pos + false_pos)), 4)

    return clf, coef_table, {"sensitivity": sensitivity}, {"specificity": specificity}, {"precision": precision}

<commit_msg>Temporarily remove logistic_regression module to finalize core package<commit_after><|endoftext|>


<commit_before>import asyncio
import logging
import time
from collections import deque
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from wstan.relay import RelayMixin
from wstan import parse_relay_request, loop, config

class WSTunClientProtocol(WebSocketClientProtocol, RelayMixin):
    POOL_MAX_SIZE = 16
    TUN_MAX_IDLE_TIMEOUT = 35  # in seconds. close tunnel on timeout
    POOL_NOM_SIZE, TUN_MIN_IDLE_TIMEOUT = round(POOL_MAX_SIZE / 2), round(TUN_MAX_IDLE_TIMEOUT / 2)
    pool = deque()

    def __init__(self):
        super().__init__()
        self.lastIdleTime = None
        self.checkTimeoutTask = None
        self.tunOpen = asyncio.Future()
        self.pool = None

    if TUN_MAX_IDLE_TIMEOUT <= 0:
        def resetTunnel(self):
            self.sendClose(1000, reason='tunnel keep-alive disabled')

        def onResetTunnel(self):
            self.sendClose(1000, reason='tunnel keep-alive disabled')

    def _clearProxy(self):
        super()._clearProxy()
        self.lastIdleTime = time.time()
        WSTunClientProtocol.addToPool(self)

    def onOpen(self):
        self.tunOpen.set_result(None)
        self.lastIdleTime = time.time()

    def onMessage(self, payload, isBinary):
        if not isBinary:
            assert payload == b'RST'
            self.onResetTunnel()
            return

        if self.tunState == self.TUN_STATE_RESETTING1:
            return

        if self._writer:
            self._writer.write(payload)

    def onClose(self, *args, **kwargs):
        super().onClose(*args, **kwargs)
        if self.pool:
            self.pool.remove(self)
            self.pool = None

    @classmethod
    @asyncio.coroutine
    def _checkTimeout(cls, tun):
        while tun.state == cls.STATE_OPEN:
            timeout = (cls.TUN_MAX_IDLE_TIMEOUT if len(cls.pool) <= cls.POOL_NOM_SIZE else
                       cls.TUN_MIN_IDLE_TIMEOUT)
            yield from asyncio.sleep(timeout)
            if (tun.tunState == cls.TUN_STATE_IDLE and
               (time.time() - tun.lastIdleTime) > timeout):
                tun.sendClose(1000, reason='tunnel idle timeout')

    @classmethod
    def addToPool(cls, tun):
        assert tun.tunState == cls.TUN_STATE_IDLE
        if len(cls.pool) >= cls.POOL_MAX_SIZE:
            tun.sendClose(1000, reason='pool is full')
        else:
            assert not tun.checkTimeoutTask
            tun.checkTimeoutTask = asyncio.async(cls._checkTimeout(tun))
            tun.pool = cls.pool
            cls.pool.append(tun)

    @classmethod
    @asyncio.coroutine
    def getOrCreate(cls):
        if WSTunClientProtocol.pool:
            logging.debug('reuse tunnel from pool (total %s)' % len(WSTunClientProtocol.pool))
            tun = WSTunClientProtocol.pool.popleft()
            tun.checkTimeoutTask.cancel()
            tun.checkTimeoutTask = None
            tun.pool = None
        else:
            tun = (yield from loop.create_connection(
                factory, config.uri_addr, config.uri_port, ssl=config.tun_ssl))[1]
            yield from asyncio.wait([tun.tunOpen], timeout=5)
        return tun

factory = WebSocketClientFactory(config.uri)
factory.protocol = WSTunClientProtocol
factory.useragent = ''
factory.autoPingInterval = 4
factory.autoPingTimeout = 3

@asyncio.coroutine
def tcp_proxy_req_handler(reader, writer):
    # handle auth method selection
    dat = yield from reader.read(257)
    if len(dat) < 2 or dat[0] != 0x05 or len(dat) != dat[1] + 2:
        logging.warning('local SOCKS v5 server got unknown request')
        return writer.close()
    writer.write(b'\x05\x00')  # \x00 == NO AUTHENTICATION REQUIRED

    # handle relay request
    try:
        dat = yield from reader.read(262)
    except ConnectionError:
        return writer.close()
    try:
        cmd, target_info = dat[1], dat[3:]
        target_addr, target_port = parse_relay_request(target_info, allow_remain=False)
    except (ValueError, IndexError):
        logging.warning('invalid SOCKS v5 relay request')
        return writer.close()
    logging.info('requested %s:%d' % (target_addr, target_port))
    if cmd != 0x01:  # CONNECT
        writer.write(b'\x05\x07\x00\x01' + b'\x00' * 6)  # \x07 == COMMAND NOT SUPPORTED
        return writer.close()

    # By accepting request before connected to target delay can be lowered (of a round-trip).
    # But SOCKS client can't get real reason when error happens (Firefox always display
    # connection reset error). Dirty solution: generate a HTML page when a HTTP request failed
    writer.write(b'\x05\x00\x00\x01' + b'\x01' * 6)  # \x00 == SUCCEEDED
    try:
        dat = yield from reader.read(WSTunClientProtocol.BUF_SIZE)
    except ConnectionError:
        dat = None
    if not dat:
        return writer.close()

    try:
        tun = yield from WSTunClientProtocol.getOrCreate()
    except (ConnectionError, OSError, TimeoutError, asyncio.TimeoutError) as e:
        logging.error('failed to establish tunnel: %s' % e)
        return writer.close()

    tun.sendMessage(target_info + dat, isBinary=True)
    tun.setProxy(reader, writer)

def main():
    server = loop.run_until_complete(
        asyncio.start_server(tcp_proxy_req_handler, config.addr, config.port))
    print('wstan client -- SOCKS v5 server listen on %s:%d' % (config.addr, config.port))
    try:
        loop.run_forever()
    finally:
        server.close()
        loop.close()
<commit_msg>Only pooled tunnel do auto-ping<commit_after>import asyncio
import logging
import time
from collections import deque
from autobahn.asyncio.websocket import WebSocketClientProtocol, WebSocketClientFactory
from wstan.relay import RelayMixin
from wstan import parse_relay_request, loop, config

# noinspection PyAttributeOutsideInit
class CustomWSClientProtocol(WebSocketClientProtocol):
    """Add auto-ping switch (dirty way)."""
    def enableAutoPing(self, interval):
        try:
            self.autoPingInterval = interval
            self.autoPingPendingCall = loop.call_later(interval, self._sendAutoPing)
        except AttributeError:
            logging.warning('failed to enable auto-ping, maybe library changed its internal method')

    def disableAutoPing(self):
        try:
            self.autoPingInterval = 0
            if self.autoPingPendingCall:
                self.autoPingPendingCall.cancel()
                self.autoPingPendingCall = None
        except AttributeError:
            logging.warning('failed to disable auto-ping')

class WSTunClientProtocol(CustomWSClientProtocol, RelayMixin):
    POOL_MAX_SIZE = 16
    TUN_MAX_IDLE_TIMEOUT = 35  # in seconds. close tunnel on timeout
    POOL_NOM_SIZE, TUN_MIN_IDLE_TIMEOUT = round(POOL_MAX_SIZE / 2), round(TUN_MAX_IDLE_TIMEOUT / 2)
    pool = deque()

    def __init__(self):
        super().__init__()
        self.lastIdleTime = None
        self.checkTimeoutTask = None
        self.tunOpen = asyncio.Future()
        self.pool = None

    if TUN_MAX_IDLE_TIMEOUT <= 0:
        def resetTunnel(self):
            self.sendClose(1000, reason='tunnel keep-alive disabled')

        def onResetTunnel(self):
            self.sendClose(1000, reason='tunnel keep-alive disabled')

    def _clearProxy(self):
        super()._clearProxy()
        self.lastIdleTime = time.time()
        WSTunClientProtocol.addToPool(self)

    def onOpen(self):
        self.tunOpen.set_result(None)
        self.lastIdleTime = time.time()

    def onMessage(self, payload, isBinary):
        if not isBinary:
            assert payload == b'RST'
            self.onResetTunnel()
            return

        if self.tunState == self.TUN_STATE_RESETTING1:
            return

        if self._writer:
            self._writer.write(payload)

    def onClose(self, *args, **kwargs):
        super().onClose(*args, **kwargs)
        if self.pool:
            self.pool.remove(self)
            self.pool = None

    @classmethod
    @asyncio.coroutine
    def _checkTimeout(cls, tun):
        while tun.state == cls.STATE_OPEN:
            timeout = (cls.TUN_MAX_IDLE_TIMEOUT if len(cls.pool) <= cls.POOL_NOM_SIZE else
                       cls.TUN_MIN_IDLE_TIMEOUT)
            yield from asyncio.sleep(timeout)
            if (tun.tunState == cls.TUN_STATE_IDLE and
               (time.time() - tun.lastIdleTime) > timeout):
                tun.sendClose(1000, reason='tunnel idle timeout')

    @classmethod
    def addToPool(cls, tun):
        assert tun.tunState == cls.TUN_STATE_IDLE
        if len(cls.pool) >= cls.POOL_MAX_SIZE:
            tun.sendClose(1000, reason='pool is full')
        else:
            assert not tun.checkTimeoutTask
            tun.checkTimeoutTask = asyncio.async(cls._checkTimeout(tun))
            tun.pool = cls.pool
            tun.enableAutoPing(4)
            cls.pool.append(tun)

    @classmethod
    @asyncio.coroutine
    def getOrCreate(cls):
        if WSTunClientProtocol.pool:
            logging.debug('reuse tunnel from pool (total %s)' % len(WSTunClientProtocol.pool))
            tun = WSTunClientProtocol.pool.popleft()
            tun.checkTimeoutTask.cancel()
            tun.checkTimeoutTask = None
            tun.pool = None
            tun.disableAutoPing()
        else:
            tun = (yield from loop.create_connection(
                factory, config.uri_addr, config.uri_port, ssl=config.tun_ssl))[1]
            yield from asyncio.wait([tun.tunOpen], timeout=5)
        return tun

factory = WebSocketClientFactory(config.uri)
factory.protocol = WSTunClientProtocol
factory.useragent = ''
factory.autoPingTimeout = 3

@asyncio.coroutine
def tcp_proxy_req_handler(reader, writer):
    # handle auth method selection
    dat = yield from reader.read(257)
    if len(dat) < 2 or dat[0] != 0x05 or len(dat) != dat[1] + 2:
        logging.warning('local SOCKS v5 server got unknown request')
        return writer.close()
    writer.write(b'\x05\x00')  # \x00 == NO AUTHENTICATION REQUIRED

    # handle relay request
    try:
        dat = yield from reader.read(262)
    except ConnectionError:
        return writer.close()
    try:
        cmd, target_info = dat[1], dat[3:]
        target_addr, target_port = parse_relay_request(target_info, allow_remain=False)
    except (ValueError, IndexError):
        logging.warning('invalid SOCKS v5 relay request')
        return writer.close()
    logging.info('requested %s:%d' % (target_addr, target_port))
    if cmd != 0x01:  # CONNECT
        writer.write(b'\x05\x07\x00\x01' + b'\x00' * 6)  # \x07 == COMMAND NOT SUPPORTED
        return writer.close()

    # By accepting request before connected to target delay can be lowered (of a round-trip).
    # But SOCKS client can't get real reason when error happens (Firefox always display
    # connection reset error). Dirty solution: generate a HTML page when a HTTP request failed
    writer.write(b'\x05\x00\x00\x01' + b'\x01' * 6)  # \x00 == SUCCEEDED
    try:
        dat = yield from reader.read(WSTunClientProtocol.BUF_SIZE)
    except ConnectionError:
        dat = None
    if not dat:
        return writer.close()

    try:
        tun = yield from WSTunClientProtocol.getOrCreate()
    except (ConnectionError, OSError, TimeoutError, asyncio.TimeoutError) as e:
        logging.error('failed to establish tunnel: %s' % e)
        return writer.close()

    tun.sendMessage(target_info + dat, isBinary=True)
    tun.setProxy(reader, writer)

def main():
    server = loop.run_until_complete(
        asyncio.start_server(tcp_proxy_req_handler, config.addr, config.port))
    print('wstan client -- SOCKS v5 server listen on %s:%d' % (config.addr, config.port))
    try:
        loop.run_forever()
    finally:
        server.close()
        loop.close()
<|endoftext|>


<commit_before>"""
    wtforms.form
    ~~~~~~~~~~~~

    TODO

    :copyright: 2007-2008 by James Crasta, Thomas Johansson.
    :license: MIT, see LICENSE.txt for details.
"""
from wtforms.validators import ValidationError
import types

class Form(object):
    def __init__(self, formdata=None, obj=None, prefix='', idprefix='', **kwargs):
        if prefix:
            prefix += '_'
        self._idprefix = idprefix

        # populate data from form and optional instance and defaults
        self.errors = {}
        self._fields = []
        has_formdata = bool(formdata)
        for name, f in self._unbound_fields:
            form_name = prefix + name
            field = f(name=form_name, form=self)
            self._fields.append((name, field))
            setattr(self, name, field)

            if name in kwargs:
                field.process_data(kwargs[name], has_formdata)
            if hasattr(obj, name):
                field.process_data(getattr(obj, name), has_formdata)
            if has_formdata and form_name in formdata:
                try:
                    data = formdata.getlist(form_name)
                except AttributeError:
                    data = formdata.getall(form_name)
                field.process_formdata(data)

    def __new__(cls, *args, **kw):
        """
        Use the field creation counter to create an ordered list of form fields.
        """
        if '_unbound_fields' not in cls.__dict__:
            fields = []
            for name in dir(cls):
                if not name.startswith('_'):
                    field = getattr(cls, name)
                    if  hasattr(field, '_formfield'):
                        fields.append((name, field))
            fields.sort(lambda x,y: cmp(x[1].creation_counter, y[1].creation_counter))
            cls._unbound_fields = fields
        return super(Form, cls).__new__(cls, *args, **kw)

    def __iter__(self):
        for name, field in self._fields:
            yield field

    def validate(self):
        success = True
        for name, field in self._fields:
            field.errors = []
            if not field.required and not field.data:
                continue
            validators = list(field.validators)
            validators.append(field._validate)
            inline_validator = getattr(self.__class__, '_validate_%s' % name, None)
            if inline_validator is not None:
                validators.append(inline_validator)
            for validator in validators:
                try:
                    validator(self, field)
                except ValueError, e:
                    field.errors.append(e.args[0])
            if field.errors:
                success = False
                self.errors[name] = field.errors
        return success

    def _get_data(self):
        data = {}
        for name, field in self._fields:
            data[name] = field.data
        return data
    data = property(_get_data)

    def auto_populate(self, model):
        """
        Automatically copy our converted form values into the model object.

        This can be very dangerous if not used properly, so make sure to only use
        this in forms with proper validators and the right attributes.
        """
        for name, field in self._fields:
            setattr(model, name, field.data)
<commit_msg>`del form.field` will now remove the field from the internal _fields list so it doesn't get validated, and assign None to form.field to mask the partial object at the class level. This is probably not the most ideal solution. Alternatives welcome.<commit_after>"""
    wtforms.form
    ~~~~~~~~~~~~

    TODO

    :copyright: 2007-2008 by James Crasta, Thomas Johansson.
    :license: MIT, see LICENSE.txt for details.
"""
from wtforms.validators import ValidationError
import types

class Form(object):
    def __init__(self, formdata=None, obj=None, prefix='', idprefix='', **kwargs):
        if prefix:
            prefix += '_'
        self._idprefix = idprefix

        # populate data from form and optional instance and defaults
        self.errors = {}
        self._fields = []
        has_formdata = bool(formdata)
        for name, f in self._unbound_fields:
            form_name = prefix + name
            field = f(name=form_name, form=self)
            self._fields.append((name, field))
            setattr(self, name, field)

            if name in kwargs:
                field.process_data(kwargs[name], has_formdata)
            if hasattr(obj, name):
                field.process_data(getattr(obj, name), has_formdata)
            if has_formdata and form_name in formdata:
                try:
                    data = formdata.getlist(form_name)
                except AttributeError:
                    data = formdata.getall(form_name)
                field.process_formdata(data)

    def __new__(cls, *args, **kw):
        """
        Use the field creation counter to create an ordered list of form fields.
        """
        if '_unbound_fields' not in cls.__dict__:
            fields = []
            for name in dir(cls):
                if not name.startswith('_'):
                    field = getattr(cls, name)
                    if  hasattr(field, '_formfield'):
                        fields.append((name, field))
            fields.sort(lambda x,y: cmp(x[1].creation_counter, y[1].creation_counter))
            cls._unbound_fields = fields
        return super(Form, cls).__new__(cls, *args, **kw)

    def __iter__(self):
        for name, field in self._fields:
            yield field

    def __delattr__(self, name):
        try:
            self._fields.remove((name, getattr(self, name)))
            setattr(self, name, None)
        except ValueError:
            super(Form, self).__delattr__(name)

    def validate(self):
        success = True
        for name, field in self._fields:
            field.errors = []
            if not field.required and not field.data:
                continue
            validators = list(field.validators)
            validators.append(field._validate)
            inline_validator = getattr(self.__class__, '_validate_%s' % name, None)
            if inline_validator is not None:
                validators.append(inline_validator)
            for validator in validators:
                try:
                    validator(self, field)
                except ValueError, e:
                    field.errors.append(e.args[0])
            if field.errors:
                success = False
                self.errors[name] = field.errors
        return success

    def _get_data(self):
        data = {}
        for name, field in self._fields:
            data[name] = field.data
        return data
    data = property(_get_data)

    def auto_populate(self, model):
        """
        Automatically copy our converted form values into the model object.

        This can be very dangerous if not used properly, so make sure to only use
        this in forms with proper validators and the right attributes.
        """
        for name, field in self._fields:
            setattr(model, name, field.data)
<|endoftext|>