<commit_before>from .context import hunter
from hunter import UnauthorizedToken
import mock
import pytest
import requests

@pytest.fixture()
def reviewsapi():
    yield hunter.ReviewsAPI()

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_certifications_list(mock_certifications, reviewsapi):
    projects_list = [{'project_id': 15, 'status': 'certified'},
                    {'project_id': 14, 'status': 'certified'}]

    mock_certifications.return_value.json.return_value = projects_list
    expected_certifications_list = [15, 14]
    certifications_list = reviewsapi.certifications()

    assert certifications_list == expected_certifications_list

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_empty_certifications_list(mock_certifications, reviewsapi):
    mock_certifications.return_value.json.return_value = []
    certifications_list = reviewsapi.certifications()

    assert certifications_list == []

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_certification_list_searching_just_for_certified_projects(mock_certifications, reviewsapi):
    projects_list = [{'project_id': 145, 'status': 'certified'},
                    {'project_id': 15, 'status': 'applied'},
                    {'project_id': 14, 'status': 'certified'}]

    mock_certifications.return_value.json.return_value = projects_list
    expected_certifications_list = [145, 14]
    certifications_list = reviewsapi.certifications()

    assert certifications_list == expected_certifications_list

@mock.patch('hunter.UnauthorizedToken')
@mock.patch('hunter.reviewsapi.requests.get')
def test_unauthorized_url_access_when_try_access_to_certifications_list(mock_certifications, mock_http_error_handler, reviewsapi):
    mock_certifications.return_value.json.side_effect = requests.exceptions.HTTPError()
    mock_http_error_handler.side_effect = UnauthorizedToken()

    with pytest.raises(UnauthorizedToken):
        reviewsapi.certifications()

@mock.patch('hunter.reviewsapi.requests.post')
def test_create_new_request_with_wanted_projects(mock_request, reviewsapi):
    mock_request.status_code = 200
    fake_certifications_list = [42, 57]
    response = reviewsapi.request_reviews(fake_certifications_list)

    assert response.status_code is not None
<commit_msg>Test if the application validates unsuccessful cases like 401<commit_after>from .context import hunter
from hunter import UnauthorizedToken
import mock
import pytest
import requests

@pytest.fixture()
def reviewsapi():
    yield hunter.ReviewsAPI()

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_certifications_list(mock_certifications, reviewsapi):
    projects_list = [{'project_id': 15, 'status': 'certified'},
                    {'project_id': 14, 'status': 'certified'}]

    mock_certifications.return_value.ok = True
    mock_certifications.return_value.json.return_value = projects_list
    expected_certifications_list = [15, 14]
    certifications_list = reviewsapi.certifications()

    assert certifications_list == expected_certifications_list

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_empty_certifications_list(mock_certifications, reviewsapi):
    mock_certifications.return_value.ok = True
    mock_certifications.return_value.json.return_value = []
    certifications_list = reviewsapi.certifications()

    assert certifications_list == []

@mock.patch('hunter.reviewsapi.requests.get')
def test_retrieve_certification_list_searching_just_for_certified_projects(mock_certifications, reviewsapi):
    projects_list = [{'project_id': 145, 'status': 'certified'},
                    {'project_id': 15, 'status': 'applied'},
                    {'project_id': 14, 'status': 'certified'}]

    mock_certifications.return_value.ok = True
    mock_certifications.return_value.json.return_value = projects_list
    expected_certifications_list = [145, 14]
    certifications_list = reviewsapi.certifications()

    assert certifications_list == expected_certifications_list

@mock.patch('hunter.UnauthorizedToken')
@mock.patch('hunter.reviewsapi.requests.get')
def test_unauthorized_url_access_when_try_access_to_certifications_list(mock_certifications, mock_http_error_handler, reviewsapi):
    mock_certifications.return_value.ok = False
    mock_certifications.return_value.json.side_effect = requests.exceptions.HTTPError()
    mock_http_error_handler.side_effect = UnauthorizedToken()

    with pytest.raises(UnauthorizedToken):
        reviewsapi.certifications()

@mock.patch('hunter.reviewsapi.requests.get')
def test_should_throw_an_exception_when_status_code_is_different_of_2xx(mock_request, reviewsapi):
    http_error = requests.exceptions.HTTPError()
    mock_request.return_value.ok = False
    mock_request.return_value.raise_for_status.side_effect = http_error

    with pytest.raises(Exception):
        reviewsapi.certifications()

@mock.patch('hunter.reviewsapi.requests.post')
def test_create_new_request_with_wanted_projects(mock_request, reviewsapi):
    mock_request.return_value.ok = True
    fake_certifications_list = [42, 57]
    response = reviewsapi.request_reviews(fake_certifications_list)

    assert response.status_code is not None
<|endoftext|>


<commit_before>import datetime
import sys

import dateutil.parser
import dateutil.tz
import pytest

from nikola.plugins.command.new_post import get_date

freezegun = pytest.importorskip('freezegun')
freeze_time = freezegun.freeze_time

UTC = dateutil.tz.tzutc()
_NOW = datetime.datetime(  # Thursday
    2013, 8, 22, 10, 0, 0, tzinfo=UTC)

@pytest.fixture(autouse=True)
def disable_six_modules():
    deleted = {
        name: sys.modules.pop(name)
        for name in sys.modules
        if name.startswith("six.moves.")
    }
    try:
        yield
    finally:
        for name, mod in deleted.items():
            sys.modules[name] = mod

@freeze_time(_NOW)
def test_get_date():
    NOW = _NOW.strftime('%Y-%m-%d %H:%M:%S %Z')
    TODAY = dateutil.parser.parse(NOW)
    RULE_TH = 'RRULE:FREQ=WEEKLY;BYDAY=TH'
    RULE_FR = 'RRULE:FREQ=WEEKLY;BYDAY=FR'

    # NOW does not match rule #########################################
    # No last date
    expected = TODAY.replace(day=23)
    assert expected == get_date(True, RULE_FR, tz=UTC)[1]
    assert expected == get_date(True, RULE_FR, tz=UTC)[1]

    # Last date in the past; doesn't match rule
    date = TODAY.replace(hour=7)
    expected = TODAY.replace(day=23, hour=7)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the future; doesn't match rule
    date = TODAY.replace(day=24, hour=7)
    expected = TODAY.replace(day=30, hour=7)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the past; matches rule
    date = TODAY.replace(day=16, hour=8)
    expected = TODAY.replace(day=23, hour=8)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the future; matches rule
    date = TODAY.replace(day=23, hour=18)
    expected = TODAY.replace(day=30, hour=18)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # NOW matches rule ################################################
    # Not scheduling should return NOW
    assert _NOW == get_date(False, RULE_TH, tz=UTC)[1]

    # No last date
    assert _NOW == get_date(True, RULE_TH, tz=UTC)[1]
    assert _NOW == get_date(True, RULE_TH, tz=UTC)[1]

    # Last date in the past; doesn't match rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=21, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=21, hour=18)
    expected = TODAY.replace(day=22, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the future; doesn't match rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=24, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=24, hour=18)
    expected = TODAY.replace(day=29, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the past; matches rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=15, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has already passed, today; rule specifies HOUR
    date = TODAY.replace(day=15, hour=7)
    expected = TODAY.replace(day=29, hour=9)
    assert expected == get_date(True, RULE_TH + ';BYHOUR=9', date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=15, hour=18)
    expected = TODAY.replace(day=22, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the future; matches rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=29, hour=7)
    expected = TODAY.replace(day=5, month=9, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=22, hour=18)
    expected = TODAY.replace(day=29, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
<commit_msg>We can also access them as globals.<commit_after>import datetime
import sys

import dateutil.parser
import dateutil.tz
import pytest

from nikola.plugins.command.new_post import get_date

freezegun = pytest.importorskip('freezegun')
freeze_time = freezegun.freeze_time

UTC = dateutil.tz.tzutc()
_NOW = datetime.datetime(  # Thursday
    2013, 8, 22, 10, 0, 0, tzinfo=UTC)
RULE_TH = 'RRULE:FREQ=WEEKLY;BYDAY=TH'
RULE_FR = 'RRULE:FREQ=WEEKLY;BYDAY=FR'

@pytest.fixture(autouse=True)
def disable_six_modules():
    deleted = {
        name: sys.modules.pop(name)
        for name in sys.modules
        if name.startswith("six.moves.")
    }
    try:
        yield
    finally:
        for name, mod in deleted.items():
            sys.modules[name] = mod

@freeze_time(_NOW)
def test_get_date():
    NOW = _NOW.strftime('%Y-%m-%d %H:%M:%S %Z')
    TODAY = dateutil.parser.parse(NOW)

    # NOW does not match rule #########################################
    # No last date
    expected = TODAY.replace(day=23)
    assert expected == get_date(True, RULE_FR, tz=UTC)[1]
    assert expected == get_date(True, RULE_FR, tz=UTC)[1]

    # Last date in the past; doesn't match rule
    date = TODAY.replace(hour=7)
    expected = TODAY.replace(day=23, hour=7)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the future; doesn't match rule
    date = TODAY.replace(day=24, hour=7)
    expected = TODAY.replace(day=30, hour=7)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the past; matches rule
    date = TODAY.replace(day=16, hour=8)
    expected = TODAY.replace(day=23, hour=8)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # Last date in the future; matches rule
    date = TODAY.replace(day=23, hour=18)
    expected = TODAY.replace(day=30, hour=18)
    assert expected == get_date(True, RULE_FR, date, tz=UTC)[1]

    # NOW matches rule ################################################
    # Not scheduling should return NOW
    assert _NOW == get_date(False, RULE_TH, tz=UTC)[1]

    # No last date
    assert _NOW == get_date(True, RULE_TH, tz=UTC)[1]
    assert _NOW == get_date(True, RULE_TH, tz=UTC)[1]

    # Last date in the past; doesn't match rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=21, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=21, hour=18)
    expected = TODAY.replace(day=22, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the future; doesn't match rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=24, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=24, hour=18)
    expected = TODAY.replace(day=29, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the past; matches rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=15, hour=7)
    expected = TODAY.replace(day=29, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has already passed, today; rule specifies HOUR
    date = TODAY.replace(day=15, hour=7)
    expected = TODAY.replace(day=29, hour=9)
    assert expected == get_date(True, RULE_TH + ';BYHOUR=9', date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=15, hour=18)
    expected = TODAY.replace(day=22, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]

    # Last date in the future; matches rule
    # Corresponding time has already passed, today
    date = TODAY.replace(day=29, hour=7)
    expected = TODAY.replace(day=5, month=9, hour=7)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
    # Corresponding time has not passed today
    date = TODAY.replace(day=22, hour=18)
    expected = TODAY.replace(day=29, hour=18)
    assert expected == get_date(True, RULE_TH, date, tz=UTC)[1]
<|endoftext|>


<commit_before>from collections import Counter
import sys
import unittest

import numpy as np

sys.path.append("../metal")
from metal.metrics import (
    accuracy_score,
    coverage_score,
)

from synthetics import (
    generate_single_task_unipolar,
)

class SyntheticsTest(unittest.TestCase):

    def check_L(self, L, n, m, k):
        self.assertTrue(L.shape[0] == n)
        self.assertTrue(L.shape[1] == m)
        self.assertTrue(L.min() >= 0)
        self.assertTrue(L.max() <= k)

    def check_class_balance(self, L, Y, class_balance):
        counter = Counter(Y.numpy())
        class_counts = np.array([v for k, v in sorted(counter.items())])
        emp_class_balance = class_counts / sum(class_counts)
        for a, b in zip(class_balance, emp_class_balance):
            self.assertLess(abs(a - b), 0.01)

    def check_accuracies(self, L, Y, m, accs):
        emp_accs = []
        for j in range(m):
            emp_accs.append(accuracy_score(Y, L[:,j], ignore_in_pred=[0]))
        for a, b in zip(accs, emp_accs):
            self.assertLess(abs(a - b), 0.01)

    def check_recalls(self, L, Y, m, k, recs, polarities):
        emp_recs = []
        for j in range(m):
            other_labels = [x for x in range(k + 1) if x != polarities[j]]
            emp_recs.append(coverage_score(Y, L[:,j], ignore_in_gold=other_labels))
        for a, b in zip(recs, emp_recs):
            self.assertLess(abs(a - b), 0.01)

    def check_all(self, L, Y, n, m, k, accs, recs, polarities, class_balance,
        lf_balance):
        self.check_class_balance(L, Y, class_balance)
        self.check_accuracies(L, Y, m, accs)
        self.check_recalls(L, Y, m, k, recs, polarities)

    def test_single_unipolar(self):
        n = 1000
        m = 10
        L, Y, metadata = generate_single_task_unipolar(
            n, m, k=2, acc=[0.6, 0.9], rec=[0.1, 0.2],
            class_balance=None, lf_balance=None, seed=1)
        self.check_all(L, Y, **metadata)

    def test_single_unipolar_nonbinary(self):
        n = 10000
        m = 50
        L, Y, metadata = generate_single_task_unipolar(
            n, m, k=5, acc=[0.6, 0.9], rec=[0.1, 0.2],
            class_balance=None, lf_balance=None, seed=1)
        self.check_all(L, Y, **metadata)

if __name__ == '__main__':
    unittest.main()<commit_msg>Add synthetics test for imbalanced classes<commit_after>from collections import Counter
import sys
import unittest

import numpy as np

sys.path.append("../metal")
from metal.metrics import (
    accuracy_score,
    coverage_score,
)

from synthetics import (
    generate_single_task_unipolar,
)

class SyntheticsTest(unittest.TestCase):

    def check_L(self, L, n, m, k):
        self.assertTrue(L.shape[0] == n)
        self.assertTrue(L.shape[1] == m)
        self.assertTrue(L.min() >= 0)
        self.assertTrue(L.max() <= k)

    def check_class_balance(self, L, Y, class_balance):
        counter = Counter(Y.numpy())
        class_counts = np.array([v for k, v in sorted(counter.items())])
        emp_class_balance = class_counts / sum(class_counts)
        for a, b in zip(class_balance, emp_class_balance):
            self.assertLess(abs(a - b), 0.01)

    def check_accuracies(self, L, Y, m, accs):
        emp_accs = []
        for j in range(m):
            emp_accs.append(accuracy_score(Y, L[:,j], ignore_in_pred=[0]))
        for a, b in zip(accs, emp_accs):
            self.assertLess(abs(a - b), 0.01)

    def check_recalls(self, L, Y, m, k, recs, polarities):
        emp_recs = []
        for j in range(m):
            other_labels = [x for x in range(k + 1) if x != polarities[j]]
            emp_recs.append(coverage_score(Y, L[:,j], ignore_in_gold=other_labels))
        for a, b in zip(recs, emp_recs):
            self.assertLess(abs(a - b), 0.01)

    def check_all(self, L, Y, n, m, k, accs, recs, polarities, class_balance,
        lf_balance):
        self.check_class_balance(L, Y, class_balance)
        self.check_accuracies(L, Y, m, accs)
        self.check_recalls(L, Y, m, k, recs, polarities)

    def test_single_unipolar(self):
        n = 1000
        m = 10
        L, Y, metadata = generate_single_task_unipolar(
            n, m, k=2, acc=[0.6, 0.9], rec=[0.1, 0.2],
            class_balance=None, lf_balance=None, seed=1)
        self.check_all(L, Y, **metadata)

    def test_single_unipolar_nonbinary(self):
        n = 10000
        m = 50
        L, Y, metadata = generate_single_task_unipolar(
            n, m, k=5, acc=[0.6, 0.9], rec=[0.1, 0.2],
            class_balance=None, lf_balance=None, seed=1)
        self.check_all(L, Y, **metadata)

    def test_single_unipolar_imbalanced(self):
        n = 10000
        m = 10
        L, Y, metadata = generate_single_task_unipolar(
            n, m, k=2, acc=[0.6, 0.9], rec=[0.1, 0.2],
            class_balance=[0.7, 0.3], lf_balance=None, seed=1)
        self.check_all(L, Y, **metadata)

if __name__ == '__main__':
    unittest.main()<|endoftext|>


<commit_before>import unittest

import textparser
from textparser import Grammar
from textparser import Sequence
from textparser import DelimitedList
from textparser import Token
from textparser import TokenizerError

def tokenize(items):
    return [Token(*item, line=1, column=2) for item in items]

class TextParserTest(unittest.TestCase):

    def test_sequence(self):
        grammar = Grammar(Sequence('NUMBER', 'WORD'))
        tokens = tokenize([
            ('NUMBER', '1.45'),
            ('WORD', 'm'),
            ('__EOF__', '')
        ])
        tree = grammar.parse(tokens)
        self.assertEqual(tree, ['1.45', 'm'])

    def test_sequence_mismatch(self):
        grammar = Grammar(Sequence('NUMBER', 'WORD'))
        tokens = tokenize([
            ('NUMBER', '1.45'),
            ('__EOF__', '')
        ])

        with self.assertRaises(textparser.Error) as cm:
            grammar.parse(tokens)

        self.assertEqual(str(cm.exception), '')

    def test_delimited_list(self):
        grammar = Grammar(DelimitedList('WORD'))

        datas = [
            ([], []),
            ([('WORD', 'foo')], ['foo']),
            ([('WORD', 'foo'), (',', ','), ('WORD', 'bar')], ['foo', 'bar'])
        ]

        for tokens, expected_tree in datas:
            tokens = tokenize(tokens + [('__EOF__', '')])
            tree = grammar.parse(tokens)
            self.assertEqual(tree, expected_tree)

    def test_tokenizer_error(self):
        datas = [
            (2, 'hej', 'he>>!<<j'),
            (0, 'a\nb\n', '>>!<<a'),
            (1, 'a\nb\n', 'a>>!<<'),
            (2, 'a\nb\n', '>>!<<b')
        ]

        for offset, string, message in datas:
            with self.assertRaises(TokenizerError) as cm:
                raise TokenizerError(0, 1, offset, string)

            self.assertEqual(
                str(cm.exception),
                'Invalid syntax at line 0, column 1: "{}"'.format(message))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Test or zero and one or more.<commit_after>import unittest

import textparser
from textparser import Grammar
from textparser import Sequence
from textparser import OneOrMore
from textparser import ZeroOrMore
from textparser import DelimitedList
from textparser import Token
from textparser import TokenizerError

def tokenize(items):
    return [Token(*item, line=1, column=2) for item in items]

class TextParserTest(unittest.TestCase):

    def test_sequence(self):
        grammar = Grammar(Sequence('NUMBER', 'WORD'))
        tokens = tokenize([
            ('NUMBER', '1.45'),
            ('WORD', 'm'),
            ('__EOF__', '')
        ])
        tree = grammar.parse(tokens)
        self.assertEqual(tree, ['1.45', 'm'])

    def test_sequence_mismatch(self):
        grammar = Grammar(Sequence('NUMBER', 'WORD'))
        tokens = tokenize([
            ('NUMBER', '1.45'),
            ('__EOF__', '')
        ])

        with self.assertRaises(textparser.Error) as cm:
            grammar.parse(tokens)

        self.assertEqual(str(cm.exception), '')

    def test_delimited_list(self):
        grammar = Grammar(DelimitedList('WORD'))

        datas = [
            ([], []),
            ([('WORD', 'foo')], ['foo']),
            ([('WORD', 'foo'), (',', ','), ('WORD', 'bar')], ['foo', 'bar'])
        ]

        for tokens, expected_tree in datas:
            tokens = tokenize(tokens + [('__EOF__', '')])
            tree = grammar.parse(tokens)
            self.assertEqual(tree, expected_tree)

    def test_zero_or_more(self):
        grammar = Grammar(ZeroOrMore('WORD'))

        datas = [
            ([], []),
            ([('WORD', 'foo')], ['foo']),
            ([('WORD', 'foo'), ('WORD', 'bar')], ['foo', 'bar'])
        ]

        for tokens, expected_tree in datas:
            tokens = tokenize(tokens + [('__EOF__', '')])
            tree = grammar.parse(tokens)
            self.assertEqual(tree, expected_tree)

    def test_one_or_more(self):
        grammar = Grammar(OneOrMore('WORD'))

        datas = [
            ([('WORD', 'foo')], ['foo']),
            ([('WORD', 'foo'), ('WORD', 'bar')], ['foo', 'bar'])
        ]

        for tokens, expected_tree in datas:
            tokens = tokenize(tokens + [('__EOF__', '')])
            tree = grammar.parse(tokens)
            self.assertEqual(tree, expected_tree)

    def test_one_or_more_mismatch(self):
        grammar = Grammar(OneOrMore('WORD'))

        datas = [
            [],
            [('NUMBER', 'foo')]
        ]

        for tokens in datas:
            tokens = tokenize(tokens + [('__EOF__', '')])

            with self.assertRaises(textparser.Error) as cm:
                grammar.parse(tokens)

            self.assertEqual(str(cm.exception), '')

    def test_tokenizer_error(self):
        datas = [
            (2, 'hej', 'he>>!<<j'),
            (0, 'a\nb\n', '>>!<<a'),
            (1, 'a\nb\n', 'a>>!<<'),
            (2, 'a\nb\n', '>>!<<b')
        ]

        for offset, string, message in datas:
            with self.assertRaises(TokenizerError) as cm:
                raise TokenizerError(0, 1, offset, string)

            self.assertEqual(
                str(cm.exception),
                'Invalid syntax at line 0, column 1: "{}"'.format(message))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>from pymbar import timeseries
import numpy as np
from pymbar.utils_for_testing import eq, skipif

try:
    import statsmodels.api as sm
    HAVE_STATSMODELS = True
except ImportError as err:
    HAVE_STATSMODELS = False

def generate_data(N=10000, K=10):
    var = np.ones(N)

    for replica in xrange(2, K + 1):
        var = np.concatenate((var, np.ones(N)))

    X = np.random.normal(np.zeros(K * N), var).reshape((K, N)) / 10.0
    Y = np.random.normal(np.zeros(K * N), var).reshape((K, N))

    energy = 10 * (X ** 2) / 2.0 + (Y ** 2) / 2.0

    return X, Y, energy

def test_statistical_inefficiency_single():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiency(X[0])
    timeseries.statisticalInefficiency(X[0], X[0])
    timeseries.statisticalInefficiency(X[0] ** 2)
    timeseries.statisticalInefficiency(X[0] ** 2, X[0] ** 2)
    timeseries.statisticalInefficiency(energy[0])
    timeseries.statisticalInefficiency(energy[0], energy[0])

    timeseries.statisticalInefficiency(X[0], X[0] ** 2)

    # TODO: Add some checks to test statistical inefficinecies are within normal range

def test_statistical_inefficiency_multiple():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiencyMultiple(X)
    timeseries.statisticalInefficiencyMultiple(X ** 2)
    timeseries.statisticalInefficiencyMultiple(X[0, :] ** 2)
    timeseries.statisticalInefficiencyMultiple(X[0:2, :] ** 2)
    timeseries.statisticalInefficiencyMultiple(energy)
    # TODO: Add some checks to test statistical inefficinecies are within normal range

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_statistical_inefficiency_fft():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiency_fft(X[0])
    timeseries.statisticalInefficiency_fft(X[0] ** 2)
    timeseries.statisticalInefficiency_fft(energy[0])

    g0 = timeseries.statisticalInefficiency_fft(X[0])
    g1 = timeseries.statisticalInefficiency(X[0])
    g2 = timeseries.statisticalInefficiency(X[0], X[0])
    g3 = timeseries.statisticalInefficiency(X[0], fft=True)
    eq(g0, g1)
    eq(g0, g2)
    eq(g0, g3)

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_statistical_inefficiency_fft_gaussian():

    # Run multiple times to get things with and without negative "spikes" at C(1)
    for i in range(5):
        x = np.random.normal(size=100000)
        g0 = timeseries.statisticalInefficiency(x, fast=False)
        g1 = timeseries.statisticalInefficiency(x, x, fast=False)
        g2 = timeseries.statisticalInefficiency_fft(x)
        g3 = timeseries.statisticalInefficiency(x, fft=True)
        eq(g0, g1, decimal=5)
        eq(g0, g2, decimal=5)
        eq(g0, g3, decimal=5)

        eq(np.log(g0), np.log(1.0), decimal=1)

    for i in range(5):
        x = np.random.normal(size=100000)
        x = np.repeat(x, 3)  # e.g. Construct correlated gaussian e.g. [a, b, c] -> [a, a, a, b, b, b, c, c, c]
        g0 = timeseries.statisticalInefficiency(x, fast=False)
        g1 = timeseries.statisticalInefficiency(x, x, fast=False)
        g2 = timeseries.statisticalInefficiency_fft(x)
        g3 = timeseries.statisticalInefficiency(x, fft=True)
        eq(g0, g1, decimal=5)
        eq(g0, g2, decimal=5)
        eq(g0, g3, decimal=5)

        eq(np.log(g0), np.log(3.0), decimal=1)

def test_detectEquil():
    x = np.random.normal(size=10000)
    (t, g, Neff_max) = timeseries.detectEquilibration(x)

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_detectEquil_binary():
    x = np.random.normal(size=10000)
    (t, g, Neff_max) = timeseries.detectEquilibration_binary_search(x)

def test_detectEquil_constant_trailing():
    # This explicitly tests issue #122, see https://github.com/choderalab/pymbar/issues/122
    x = np.random.normal(size=100) * 0.01
    x[50:] = 3.0
    # The input data is some MCMC chain where the trailing end of the chain is a constant sequence.
    (t, g, Neff_max) = timeseries.detectEquilibration(x)
    # We only check that the code doesn't give an exception.
<commit_msg>Don't check exact numbers in test.<commit_after>from pymbar import timeseries
import numpy as np
from pymbar.utils_for_testing import eq, skipif

try:
    import statsmodels.api as sm
    HAVE_STATSMODELS = True
except ImportError as err:
    HAVE_STATSMODELS = False

def generate_data(N=10000, K=10):
    var = np.ones(N)

    for replica in xrange(2, K + 1):
        var = np.concatenate((var, np.ones(N)))

    X = np.random.normal(np.zeros(K * N), var).reshape((K, N)) / 10.0
    Y = np.random.normal(np.zeros(K * N), var).reshape((K, N))

    energy = 10 * (X ** 2) / 2.0 + (Y ** 2) / 2.0

    return X, Y, energy

def test_statistical_inefficiency_single():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiency(X[0])
    timeseries.statisticalInefficiency(X[0], X[0])
    timeseries.statisticalInefficiency(X[0] ** 2)
    timeseries.statisticalInefficiency(X[0] ** 2, X[0] ** 2)
    timeseries.statisticalInefficiency(energy[0])
    timeseries.statisticalInefficiency(energy[0], energy[0])

    timeseries.statisticalInefficiency(X[0], X[0] ** 2)

    # TODO: Add some checks to test statistical inefficinecies are within normal range

def test_statistical_inefficiency_multiple():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiencyMultiple(X)
    timeseries.statisticalInefficiencyMultiple(X ** 2)
    timeseries.statisticalInefficiencyMultiple(X[0, :] ** 2)
    timeseries.statisticalInefficiencyMultiple(X[0:2, :] ** 2)
    timeseries.statisticalInefficiencyMultiple(energy)
    # TODO: Add some checks to test statistical inefficinecies are within normal range

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_statistical_inefficiency_fft():
    X, Y, energy = generate_data()
    timeseries.statisticalInefficiency_fft(X[0])
    timeseries.statisticalInefficiency_fft(X[0] ** 2)
    timeseries.statisticalInefficiency_fft(energy[0])

    g0 = timeseries.statisticalInefficiency_fft(X[0])
    g1 = timeseries.statisticalInefficiency(X[0])
    g2 = timeseries.statisticalInefficiency(X[0], X[0])
    g3 = timeseries.statisticalInefficiency(X[0], fft=True)
    eq(g0, g1)
    eq(g0, g2)
    eq(g0, g3)

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_statistical_inefficiency_fft_gaussian():

    # Run multiple times to get things with and without negative "spikes" at C(1)
    for i in range(5):
        x = np.random.normal(size=100000)
        g0 = timeseries.statisticalInefficiency(x, fast=False)
        g1 = timeseries.statisticalInefficiency(x, x, fast=False)
        g2 = timeseries.statisticalInefficiency_fft(x)
        g3 = timeseries.statisticalInefficiency(x, fft=True)
        eq(g0, g1, decimal=5)
        eq(g0, g2, decimal=5)
        eq(g0, g3, decimal=5)

        eq(np.log(g0), np.log(1.0), decimal=1)

    for i in range(5):
        x = np.random.normal(size=100000)
        x = np.repeat(x, 3)  # e.g. Construct correlated gaussian e.g. [a, b, c] -> [a, a, a, b, b, b, c, c, c]
        g0 = timeseries.statisticalInefficiency(x, fast=False)
        g1 = timeseries.statisticalInefficiency(x, x, fast=False)
        g2 = timeseries.statisticalInefficiency_fft(x)
        g3 = timeseries.statisticalInefficiency(x, fft=True)
        eq(g0, g1, decimal=5)
        eq(g0, g2, decimal=5)
        eq(g0, g3, decimal=5)

        eq(np.log(g0), np.log(3.0), decimal=1)

def test_detectEquil():
    x = np.random.normal(size=10000)
    (t, g, Neff_max) = timeseries.detectEquilibration(x)

@skipif(not HAVE_STATSMODELS, "Skipping FFT based tests because statsmodels not installed.")
def test_detectEquil_binary():
    x = np.random.normal(size=10000)
    (t, g, Neff_max) = timeseries.detectEquilibration_binary_search(x)

def test_detectEquil_constant_trailing():
    # This explicitly tests issue #122, see https://github.com/choderalab/pymbar/issues/122
    x = np.random.normal(size=100) * 0.01
    x[50:] = 3.0
    # The input data is some MCMC chain where the trailing end of the chain is a constant sequence.
    (t, g, Neff_max) = timeseries.detectEquilibration(x)
    """
    We only check that the code doesn't give an exception.  The exact value of Neff can either be
    ~50 if we try to include part of the equilibration samples, or it can be Neff=1 if we find that the
    whole first half is discarded.
<|endoftext|>


<commit_before>import os
import json
import uuid
import shutil
import asyncio
import hashlib

from waterbutler.core import utils
from waterbutler.core import signing
from waterbutler.core import streams
from waterbutler.core import provider
from waterbutler.core import exceptions
from waterbutler.core.path import WaterButlerPath

from waterbutler.providers.osfstorage import settings
from waterbutler.providers.osfstorage.tasks import backup
from waterbutler.providers.osfstorage.tasks import parity
from waterbutler.providers.osfstorage.metadata import OsfStorageFileMetadata
from waterbutler.providers.osfstorage.metadata import OsfStorageFolderMetadata
from waterbutler.providers.osfstorage.metadata import OsfStorageRevisionMetadata

QUERY_METHODS = ('GET', 'DELETE')

class OSFPath(utils.WaterButlerPath):
    def __init__(self, path):
        super().__init__(path, prefix=True, suffix=True)

class OSFStorageProvider(provider.BaseProvider):
    __version__ = '0.0.1'

    NAME = 'osfstorage'

    def __init__(self, auth, credentials, settings):
        super().__init__(auth, credentials, settings)
        self.nid = settings['nid']
        self.root_id = settings['rootId']
        self.BASE_URL = settings['baseUrl']
        # BASE_URL = settings.BASE_URL
        # self.copy_url = settings.get('copy')
        # self.move_url = settings.get('move')
        # self.callback_url = settings.get('callback')
        # self.metadata_url = settings.get('metadata')
        # self.revisions_url = settings.get('revisions')
        # self.lineage_url = settings.get('lineage')
        # self.create_folder_url = settings.get('createFolder')
        self.provider_name = settings['storage'].get('provider')

        self.parity_settings = settings.get('parity')
        self.parity_credentials = credentials.get('parity')

        self.archive_settings = settings.get('archive')
        self.archive_credentials = credentials.get('archive')

    @asyncio.coroutine
    def validate_path(self, path, **kwargs):
        if path == '/':
            return WaterButlerPath('/', _ids=[self.root_id], folder=True)

        try:
            path, name = path.strip('/').split('/')
        except ValueError:
            path, name = path, None

        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path, 'lineage'),
            expects=(200, 404)
        )

        if resp.status == 404:
            return WaterButlerPath(path, _ids=[self.root_id, None], folder=True)

        data = yield from resp.json()

        names, ids = zip(*[(x['name'], x['path']) for x in reversed(data['data'])])
        if name is not None:
            ids += (None, )
            names += (name, )

        return WaterButlerPath('/'.join(names), _ids=ids, folder='folder' == data['data'][0]['kind'])

    def revalidate_path(self, base, path, folder=False):
        assert base.is_dir

        try:
            data = next(
                x for x in
                (yield from self.metadata(base))
                if x['name'] == path and
                x['kind'] == ('folder' if folder else 'file')
            )

            return base.child(data['name'], _id=data['path'].strip('/'), folder=folder)
        except StopIteration:
            return base.child(path, folder=folder)

    def make_provider(self, settings):
        """Requests on different files may need to use different providers,
        instances, e.g. when different files lives in different containers
        within a provider. This helper creates a single-use provider instance
        that optionally overrides the settings.

        :param dict settings: Overridden settings
        """
        return utils.make_provider(
            self.provider_name,
            self.auth,
            self.credentials['storage'],
            self.settings['storage'],
        )

    def can_intra_copy(self, other, path=None):
        return isinstance(other, self.__class__)

    def can_intra_move(self, other, path=None):
        return isinstance(other, self.__class__)

    def intra_move(self, dest_provider, src_path, dest_path):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url('hooks', 'move'),
            data=json.dumps({
                'user': self.auth['id'],
                'source': src_path.identifier,
                'destination': {
                    'name': dest_path.name,
                    'node': dest_provider.nid,
                    'parent': dest_path.parent.identifier
                }
            }),
            headers={'Content-Type': 'application/json'},
            expects=(200, 201)
        )

        data = yield from resp.json()

        if data['kind'] == 'file':
            return OsfStorageFileMetadata(data).serialized(), resp.status == 201

        return OsfStorageFolderMetadata(data).serialized(), resp.status == 201

    def intra_copy(self, dest_provider, src_path, dest_path):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url('hooks', 'copy'),
            data=json.dumps({
                'user': self.auth['id'],
                'source': src_path.identifier,
                'destination': {
                    'name': dest_path.name,
                    'node': dest_provider.nid,
                    'parent': dest_path.parent.identifier
                }
            }),
            headers={'Content-Type': 'application/json'},
            expects=(200, 201)
        )

        data = yield from resp.json()

        if data['kind'] == 'file':
            return OsfStorageFileMetadata(data).serialized(), resp.status == 201

        return OsfStorageFolderMetadata(data).serialized(), resp.status == 201

    @asyncio.coroutine
    def make_signed_request(self, method, url, data=None, params=None, ttl=100, **kwargs):
        signer = signing.Signer(settings.HMAC_SECRET, settings.HMAC_ALGORITHM)
        if method.upper() in QUERY_METHODS:
            signed = signing.sign_data(signer, params or {}, ttl=ttl)
            params = signed
        else:
            signed = signing.sign_data(signer, json.loads(data or {}), ttl=ttl)
            data = json.dumps(signed)
        return (yield from self.make_request(method, url + '/', data=data, params=params, **kwargs))

    @asyncio.coroutine
    def download(self, path, version=None, **kwargs):
        # osf storage metadata will return a virtual path within the provider
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'download', version=version),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )

        data = yield from resp.json()
        provider = self.make_provider(data['settings'])
        name = data['data'].pop('name')
        data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
        download_kwargs = {}
        download_kwargs.update(kwargs)
        download_kwargs.update(data['data'])
        download_kwargs['displayName'] = kwargs.get('displayName', name)
        return (yield from provider.download(**download_kwargs))

    @asyncio.coroutine
    def upload(self, stream, path, **kwargs):
        self._create_paths()

        pending_name = str(uuid.uuid4())
        provider = self.make_provider(self.settings)
        local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
        remote_pending_path = yield from provider.validate_path('/' + pending_name)

        stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
        stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
        stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))

        with open(local_pending_path, 'wb') as file_pointer:
            stream.add_writer('file', file_pointer)
            yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)

        complete_name = stream.writers['sha256'].hexdigest
        local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
        remote_complete_path = yield from provider.validate_path('/' + complete_name)

        try:
            metadata = yield from provider.metadata(remote_complete_path)
        except exceptions.MetadataError as e:
            if e.code != 404:
                raise
            metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
        else:
            yield from provider.delete(remote_pending_path)

        # Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
        # http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
        shutil.move(local_pending_path, local_complete_path)

        response = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            expects=(200, 201),
            data=json.dumps({
                'name': path.name,
                'user': self.auth['id'],
                'settings': self.settings['storage'],
                'metadata': metadata,
                'hashes': {
                    'md5': stream.writers['md5'].hexdigest,
                    'sha1': stream.writers['sha1'].hexdigest,
                    'sha256': stream.writers['sha256'].hexdigest,
                },
                'worker': {
                    'host': os.uname()[1],
                    # TODO: Include additional information
                    'address': None,
                    'version': self.__version__,
                },
            }),
            headers={'Content-Type': 'application/json'},
        )

        created = response.status == 201
        data = yield from response.json()

        if settings.RUN_TASKS:
            version_id = data['version']
            parity.main(
                local_complete_path,
                self.parity_credentials,
                self.parity_settings,
            )
            backup.main(
                local_complete_path,
                version_id,
                self.callback_url,
                self.archive_credentials,
                self.archive_settings,
            )

        name = path.name

        metadata.update({
            'name': name,
            'path': data['path'],
            'version': data['version'],
            'downloads': data['downloads']
        })

        return OsfStorageFileMetadata(metadata).serialized(), created

    @asyncio.coroutine
    def delete(self, path, **kwargs):
        if path.identifier is None:
            raise exceptions.MetadataError('{} not found'.format(str(path)), code=404)

        yield from self.make_signed_request(
            'DELETE',
            self.build_url(path.identifier),
            params={'user': self.auth['id']},
            expects=(200, )
        )

    @asyncio.coroutine
    def metadata(self, path, **kwargs):
        if path.identifier is None:
            raise exceptions.MetadataError('{} not found'.format(str(path)), code=404)

        if not path.is_dir:
            return (yield from self._item_metadata(path))
        return (yield from self._children_metadata(path))

    @asyncio.coroutine
    def revisions(self, path, **kwargs):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.parent.identifier, 'revisions'),
            expects=(200, )
        )

        return [
            OsfStorageRevisionMetadata(item).serialized()
            for item in (yield from resp.json())['revisions']
        ]

    @asyncio.coroutine
    def create_folder(self, path, **kwargs):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            data=json.dumps({
                'kind': 'folder',
                'name': path.name,
                'user': self.auth['id'],
            }),
            headers={'Content-Type': 'application/json'},
            expects=(201, )
        )

        return OsfStorageFolderMetadata(
            (yield from resp.json())
        ).serialized()

    @asyncio.coroutine
    def _item_metadata(self, path):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier),
            expects=(200, )
        )

        return OsfStorageFileMetadata((yield from resp.json())).serialized()

    @asyncio.coroutine
    def _children_metadata(self, path):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'children'),
            expects=(200, )
        )
        resp_json = yield from resp.json()

        ret = []
        for item in resp_json:
            if item['kind'] == 'folder':
                ret.append(OsfStorageFolderMetadata(item).serialized())
            else:
                ret.append(OsfStorageFileMetadata(item).serialized())
        return ret

    def _create_paths(self):
        try:
            os.mkdir(settings.FILE_PATH_PENDING)
        except FileExistsError:
            pass

        try:
            os.mkdir(settings.FILE_PATH_COMPLETE)
        except FileExistsError:
            pass

        return True
<commit_msg>Remove unused code<commit_after>import os
import json
import uuid
import shutil
import asyncio
import hashlib

from waterbutler.core import utils
from waterbutler.core import signing
from waterbutler.core import streams
from waterbutler.core import provider
from waterbutler.core import exceptions
from waterbutler.core.path import WaterButlerPath

from waterbutler.providers.osfstorage import settings
from waterbutler.providers.osfstorage.tasks import backup
from waterbutler.providers.osfstorage.tasks import parity
from waterbutler.providers.osfstorage.metadata import OsfStorageFileMetadata
from waterbutler.providers.osfstorage.metadata import OsfStorageFolderMetadata
from waterbutler.providers.osfstorage.metadata import OsfStorageRevisionMetadata

QUERY_METHODS = ('GET', 'DELETE')

class OSFPath(utils.WaterButlerPath):
    def __init__(self, path):
        super().__init__(path, prefix=True, suffix=True)

class OSFStorageProvider(provider.BaseProvider):
    __version__ = '0.0.1'

    NAME = 'osfstorage'

    def __init__(self, auth, credentials, settings):
        super().__init__(auth, credentials, settings)
        self.nid = settings['nid']
        self.root_id = settings['rootId']
        self.BASE_URL = settings['baseUrl']
        self.provider_name = settings['storage'].get('provider')

        self.parity_settings = settings.get('parity')
        self.parity_credentials = credentials.get('parity')

        self.archive_settings = settings.get('archive')
        self.archive_credentials = credentials.get('archive')

    @asyncio.coroutine
    def validate_path(self, path, **kwargs):
        if path == '/':
            return WaterButlerPath('/', _ids=[self.root_id], folder=True)

        try:
            path, name = path.strip('/').split('/')
        except ValueError:
            path, name = path, None

        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path, 'lineage'),
            expects=(200, 404)
        )

        if resp.status == 404:
            return WaterButlerPath(path, _ids=[self.root_id, None], folder=True)

        data = yield from resp.json()

        names, ids = zip(*[(x['name'], x['path']) for x in reversed(data['data'])])
        if name is not None:
            ids += (None, )
            names += (name, )

        return WaterButlerPath('/'.join(names), _ids=ids, folder='folder' == data['data'][0]['kind'])

    def revalidate_path(self, base, path, folder=False):
        assert base.is_dir

        try:
            data = next(
                x for x in
                (yield from self.metadata(base))
                if x['name'] == path and
                x['kind'] == ('folder' if folder else 'file')
            )

            return base.child(data['name'], _id=data['path'].strip('/'), folder=folder)
        except StopIteration:
            return base.child(path, folder=folder)

    def make_provider(self, settings):
        """Requests on different files may need to use different providers,
        instances, e.g. when different files lives in different containers
        within a provider. This helper creates a single-use provider instance
        that optionally overrides the settings.

        :param dict settings: Overridden settings
        """
        return utils.make_provider(
            self.provider_name,
            self.auth,
            self.credentials['storage'],
            self.settings['storage'],
        )

    def can_intra_copy(self, other, path=None):
        return isinstance(other, self.__class__)

    def can_intra_move(self, other, path=None):
        return isinstance(other, self.__class__)

    def intra_move(self, dest_provider, src_path, dest_path):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url('hooks', 'move'),
            data=json.dumps({
                'user': self.auth['id'],
                'source': src_path.identifier,
                'destination': {
                    'name': dest_path.name,
                    'node': dest_provider.nid,
                    'parent': dest_path.parent.identifier
                }
            }),
            headers={'Content-Type': 'application/json'},
            expects=(200, 201)
        )

        data = yield from resp.json()

        if data['kind'] == 'file':
            return OsfStorageFileMetadata(data).serialized(), resp.status == 201

        return OsfStorageFolderMetadata(data).serialized(), resp.status == 201

    def intra_copy(self, dest_provider, src_path, dest_path):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url('hooks', 'copy'),
            data=json.dumps({
                'user': self.auth['id'],
                'source': src_path.identifier,
                'destination': {
                    'name': dest_path.name,
                    'node': dest_provider.nid,
                    'parent': dest_path.parent.identifier
                }
            }),
            headers={'Content-Type': 'application/json'},
            expects=(200, 201)
        )

        data = yield from resp.json()

        if data['kind'] == 'file':
            return OsfStorageFileMetadata(data).serialized(), resp.status == 201

        return OsfStorageFolderMetadata(data).serialized(), resp.status == 201

    @asyncio.coroutine
    def make_signed_request(self, method, url, data=None, params=None, ttl=100, **kwargs):
        signer = signing.Signer(settings.HMAC_SECRET, settings.HMAC_ALGORITHM)
        if method.upper() in QUERY_METHODS:
            signed = signing.sign_data(signer, params or {}, ttl=ttl)
            params = signed
        else:
            signed = signing.sign_data(signer, json.loads(data or {}), ttl=ttl)
            data = json.dumps(signed)
        return (yield from self.make_request(method, url + '/', data=data, params=params, **kwargs))

    @asyncio.coroutine
    def download(self, path, version=None, **kwargs):
        # osf storage metadata will return a virtual path within the provider
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'download', version=version),
            expects=(200, ),
            throws=exceptions.DownloadError,
        )

        data = yield from resp.json()
        provider = self.make_provider(data['settings'])
        name = data['data'].pop('name')
        data['data']['path'] = yield from provider.validate_path('/' + data['data']['path'])
        download_kwargs = {}
        download_kwargs.update(kwargs)
        download_kwargs.update(data['data'])
        download_kwargs['displayName'] = kwargs.get('displayName', name)
        return (yield from provider.download(**download_kwargs))

    @asyncio.coroutine
    def upload(self, stream, path, **kwargs):
        self._create_paths()

        pending_name = str(uuid.uuid4())
        provider = self.make_provider(self.settings)
        local_pending_path = os.path.join(settings.FILE_PATH_PENDING, pending_name)
        remote_pending_path = yield from provider.validate_path('/' + pending_name)

        stream.add_writer('md5', streams.HashStreamWriter(hashlib.md5))
        stream.add_writer('sha1', streams.HashStreamWriter(hashlib.sha1))
        stream.add_writer('sha256', streams.HashStreamWriter(hashlib.sha256))

        with open(local_pending_path, 'wb') as file_pointer:
            stream.add_writer('file', file_pointer)
            yield from provider.upload(stream, remote_pending_path, check_created=False, fetch_metadata=False, **kwargs)

        complete_name = stream.writers['sha256'].hexdigest
        local_complete_path = os.path.join(settings.FILE_PATH_COMPLETE, complete_name)
        remote_complete_path = yield from provider.validate_path('/' + complete_name)

        try:
            metadata = yield from provider.metadata(remote_complete_path)
        except exceptions.MetadataError as e:
            if e.code != 404:
                raise
            metadata, _ = yield from provider.move(provider, remote_pending_path, remote_complete_path)
        else:
            yield from provider.delete(remote_pending_path)

        # Due to cross volume movement in unix we leverage shutil.move which properly handles this case.
        # http://bytes.com/topic/python/answers/41652-errno-18-invalid-cross-device-link-using-os-rename#post157964
        shutil.move(local_pending_path, local_complete_path)

        response = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            expects=(200, 201),
            data=json.dumps({
                'name': path.name,
                'user': self.auth['id'],
                'settings': self.settings['storage'],
                'metadata': metadata,
                'hashes': {
                    'md5': stream.writers['md5'].hexdigest,
                    'sha1': stream.writers['sha1'].hexdigest,
                    'sha256': stream.writers['sha256'].hexdigest,
                },
                'worker': {
                    'host': os.uname()[1],
                    # TODO: Include additional information
                    'address': None,
                    'version': self.__version__,
                },
            }),
            headers={'Content-Type': 'application/json'},
        )

        created = response.status == 201
        data = yield from response.json()

        if settings.RUN_TASKS:
            version_id = data['version']
            parity.main(
                local_complete_path,
                self.parity_credentials,
                self.parity_settings,
            )
            backup.main(
                local_complete_path,
                version_id,
                self.callback_url,
                self.archive_credentials,
                self.archive_settings,
            )

        name = path.name

        metadata.update({
            'name': name,
            'path': data['path'],
            'version': data['version'],
            'downloads': data['downloads']
        })

        return OsfStorageFileMetadata(metadata).serialized(), created

    @asyncio.coroutine
    def delete(self, path, **kwargs):
        if path.identifier is None:
            raise exceptions.MetadataError('{} not found'.format(str(path)), code=404)

        yield from self.make_signed_request(
            'DELETE',
            self.build_url(path.identifier),
            params={'user': self.auth['id']},
            expects=(200, )
        )

    @asyncio.coroutine
    def metadata(self, path, **kwargs):
        if path.identifier is None:
            raise exceptions.MetadataError('{} not found'.format(str(path)), code=404)

        if not path.is_dir:
            return (yield from self._item_metadata(path))
        return (yield from self._children_metadata(path))

    @asyncio.coroutine
    def revisions(self, path, **kwargs):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.parent.identifier, 'revisions'),
            expects=(200, )
        )

        return [
            OsfStorageRevisionMetadata(item).serialized()
            for item in (yield from resp.json())['revisions']
        ]

    @asyncio.coroutine
    def create_folder(self, path, **kwargs):
        resp = yield from self.make_signed_request(
            'POST',
            self.build_url(path.parent.identifier, 'children'),
            data=json.dumps({
                'kind': 'folder',
                'name': path.name,
                'user': self.auth['id'],
            }),
            headers={'Content-Type': 'application/json'},
            expects=(201, )
        )

        return OsfStorageFolderMetadata(
            (yield from resp.json())
        ).serialized()

    @asyncio.coroutine
    def _item_metadata(self, path):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier),
            expects=(200, )
        )

        return OsfStorageFileMetadata((yield from resp.json())).serialized()

    @asyncio.coroutine
    def _children_metadata(self, path):
        resp = yield from self.make_signed_request(
            'GET',
            self.build_url(path.identifier, 'children'),
            expects=(200, )
        )
        resp_json = yield from resp.json()

        ret = []
        for item in resp_json:
            if item['kind'] == 'folder':
                ret.append(OsfStorageFolderMetadata(item).serialized())
            else:
                ret.append(OsfStorageFileMetadata(item).serialized())
        return ret

    def _create_paths(self):
        try:
            os.mkdir(settings.FILE_PATH_PENDING)
        except FileExistsError:
            pass

        try:
            os.mkdir(settings.FILE_PATH_COMPLETE)
        except FileExistsError:
            pass

        return True
<|endoftext|>


<commit_before>#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytz
import numpy as np
import pandas as pd

from datetime import timedelta, datetime
from unittest import TestCase

from six.moves import range

from zipline.utils.test_utils import setup_logger

from zipline.protocol import Event
from zipline.sources import SpecificEquityTrades
from zipline.transforms.utils import StatefulTransform, EventWindow
from zipline.transforms import MovingVWAP
from zipline.transforms import MovingAverage
from zipline.transforms import MovingStandardDev
from zipline.transforms import Returns
import zipline.utils.factory as factory

from zipline.test_algorithms import TALIBAlgorithm

def to_dt(msg):
    return Event({'dt': msg})

class NoopEventWindow(EventWindow):
    """
    A no-op EventWindow subclass for testing the base EventWindow logic.
    Keeps lists of all added and dropped events.
    """
    def __init__(self, market_aware, days, delta):
        EventWindow.__init__(self, market_aware, days, delta)

        self.added = []
        self.removed = []

    def handle_add(self, event):
        self.added.append(event)

    def handle_remove(self, event):
        self.removed.append(event)

class TestEventWindow(TestCase):
    def setUp(self):
        self.sim_params = factory.create_simulation_parameters()

        setup_logger(self)

        self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc)
        self.eleven_normal_days = [self.monday + i * timedelta(days=1)
                                   for i in range(11)]

        # Modify the end of the period slightly to exercise the
        # incomplete day logic.
        self.eleven_normal_days[-1] -= timedelta(minutes=1)
        self.eleven_normal_days.append(self.monday +
                                       timedelta(days=11, seconds=1))

        # Second set of dates to test holiday handling.
        self.jul4_monday = datetime(2012, 7, 2, 16, tzinfo=pytz.utc)
        self.week_of_jul4 = [self.jul4_monday + i * timedelta(days=1)
                             for i in range(5)]

    def test_market_aware_window_normal_week(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=3
        )
        events = [to_dt(date) for date in self.eleven_normal_days]
        lengths = []
        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        # The window stretches out during the weekend because we wait
        # to drop events until the weekend ends. The last window is
        # briefly longer because it doesn't complete a full day.  The
        # window then shrinks once the day completes
        self.assertEquals(lengths, [1, 2, 3, 3, 3, 4, 5, 5, 5, 3, 4, 3])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-3])

    def test_market_aware_window_holiday(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=2
        )
        events = [to_dt(date) for date in self.week_of_jul4]
        lengths = []

        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        self.assertEquals(lengths, [1, 2, 3, 3, 2])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-2])

    def tearDown(self):
        setup_logger(self)

class TestFinanceTransforms(TestCase):

    def setUp(self):
        self.sim_params = factory.create_simulation_parameters()
        setup_logger(self)

        trade_history = factory.create_trade_history(
            133,
            [10.0, 10.0, 11.0, 11.0],
            [100, 100, 100, 300],
            timedelta(days=1),
            self.sim_params
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

    def tearDown(self):
        self.log_handler.pop_application()

    def test_vwap(self):
        vwap = MovingVWAP(
            market_aware=True,
            window_length=2
        )
        transformed = list(vwap.transform(self.source))

        # Output values
        tnfm_vals = [message[vwap.get_hash()] for message in transformed]
        # "Hand calculated" values.
        expected = [
            (10.0 * 100) / 100.0,
            ((10.0 * 100) + (10.0 * 100)) / (200.0),
            # We should drop the first event here.
            ((10.0 * 100) + (11.0 * 100)) / (200.0),
            # We should drop the second event here.
            ((11.0 * 100) + (11.0 * 300)) / (400.0)
        ]

        # Output should match the expected.
        self.assertEquals(tnfm_vals, expected)

    def test_returns(self):
        # Daily returns.
        returns = Returns(1)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        # No returns for the first event because we don't have a
        # previous close.
        expected = [0.0, 0.0, 0.1, 0.0]

        self.assertEquals(tnfm_vals, expected)

        # Two-day returns.  An extra kink here is that the
        # factory will automatically skip a weekend for the
        # last event. Results shouldn't notice this blip.

        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0, 13.0],
            [100, 100, 100, 300, 100],
            timedelta(days=1),
            self.sim_params
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        returns = StatefulTransform(Returns, 2)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        expected = [
            0.0,
            0.0,
            (13.0 - 10.0) / 10.0,
            (12.0 - 15.0) / 15.0,
            (13.0 - 13.0) / 13.0
        ]

        self.assertEquals(tnfm_vals, expected)

    def test_moving_average(self):

        mavg = MovingAverage(
            market_aware=True,
            fields=['price', 'volume'],
            window_length=2
        )

        transformed = list(mavg.transform(self.source))
        # Output values.
        tnfm_prices = [message[mavg.get_hash()].price
                       for message in transformed]
        tnfm_volumes = [message[mavg.get_hash()].volume
                        for message in transformed]

        # "Hand-calculated" values
        expected_prices = [
            ((10.0) / 1.0),
            ((10.0 + 10.0) / 2.0),
            # First event should get dropped here.
            ((10.0 + 11.0) / 2.0),
            # Second event should get dropped here.
            ((11.0 + 11.0) / 2.0)
        ]
        expected_volumes = [
            ((100.0) / 1.0),
            ((100.0 + 100.0) / 2.0),
            # First event should get dropped here.
            ((100.0 + 100.0) / 2.0),
            # Second event should get dropped here.
            ((100.0 + 300.0) / 2.0)
        ]

        self.assertEquals(tnfm_prices, expected_prices)
        self.assertEquals(tnfm_volumes, expected_volumes)

    def test_moving_stddev(self):
        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0],
            [100, 100, 100, 100],
            timedelta(days=1),
            self.sim_params
        )

        stddev = MovingStandardDev(
            market_aware=True,
            window_length=3,
        )

        self.source = SpecificEquityTrades(event_list=trade_history)

        transformed = list(stddev.transform(self.source))

        vals = [message[stddev.get_hash()] for message in transformed]

        expected = [
            None,
            np.std([10.0, 15.0], ddof=1),
            np.std([10.0, 15.0, 13.0], ddof=1),
            np.std([15.0, 13.0, 12.0], ddof=1),
        ]

        # np has odd rounding behavior, cf.
        # http://docs.scipy.org/doc/np/reference/generated/np.std.html
        for v1, v2 in zip(vals, expected):

            if v1 is None:
                self.assertIsNone(v2)
                continue
            self.assertEquals(round(v1, 5), round(v2, 5))

############################################################
# Test TALIB

import talib
import zipline.transforms.ta as ta

class TestTALIB(TestCase):
    def setUp(self):
        setup_logger(self)
        sim_params = factory.create_simulation_parameters(
            start=datetime(1990, 1, 1, tzinfo=pytz.utc),
            end=datetime(1990, 3, 30, tzinfo=pytz.utc))
        self.source, self.panel = \
            factory.create_test_panel_ohlc_source(sim_params)

    def test_talib_with_default_params(self):
        BLACKLIST = ['make_transform', 'BatchTransform',
                     # TODO: Figure out why MAVP generates a KeyError
                     'MAVP']
        names = [n for n in dir(ta) if n[0].isupper()
                 and n not in BLACKLIST]

        for name in names:
            print(name)
            zipline_transform = getattr(ta, name)(sid=0)
            talib_fn = getattr(talib.abstract, name)

            start = datetime(1990, 1, 1, tzinfo=pytz.utc)
            end = start + timedelta(days=zipline_transform.lookback + 10)
            sim_params = factory.create_simulation_parameters(
                start=start, end=end)
            source, panel = \
                factory.create_test_panel_ohlc_source(sim_params)

            algo = TALIBAlgorithm(talib=zipline_transform)
            algo.run(source)

            zipline_result = np.array(
                algo.talib_results[zipline_transform][-1])

            talib_data = dict()
            data = zipline_transform.window
            # TODO: Figure out if we are clobbering the tests by this
            # protection against empty windows
            if not data:
                continue
            for key in ['open', 'high', 'low', 'volume']:
                if key in data:
                    talib_data[key] = data[key][0].values
            talib_data['close'] = data['price'][0].values
            expected_result = talib_fn(talib_data)

            if isinstance(expected_result, list):
                expected_result = np.array([e[-1] for e in expected_result])
            else:
                expected_result = np.array(expected_result[-1])
            if not (np.all(np.isnan(zipline_result))
                    and np.all(np.isnan(expected_result))):
                self.assertTrue(np.allclose(zipline_result, expected_result))
            else:
                print('--- NAN')

            # reset generator so next iteration has data
            # self.source, self.panel = \
                # factory.create_test_panel_ohlc_source(self.sim_params)

    def test_multiple_talib_with_args(self):
        zipline_transforms = [ta.MA(timeperiod=10),
                              ta.MA(timeperiod=25)]
        talib_fn = talib.abstract.MA
        algo = TALIBAlgorithm(talib=zipline_transforms)
        algo.run(self.source)
        # Test if computed values match those computed by pandas rolling mean.
        sid = 0
        talib_values = np.array([x[sid] for x in
                                 algo.talib_results[zipline_transforms[0]]])
        np.testing.assert_array_equal(talib_values,
                                      pd.rolling_mean(self.panel[0]['price'],
                                                      10).values)
        talib_values = np.array([x[sid] for x in
                                 algo.talib_results[zipline_transforms[1]]])
        np.testing.assert_array_equal(talib_values,
                                      pd.rolling_mean(self.panel[0]['price'],
                                                      25).values)
        for t in zipline_transforms:
            talib_result = np.array(algo.talib_results[t][-1])
            talib_data = dict()
            data = t.window
            # TODO: Figure out if we are clobbering the tests by this
            # protection against empty windows
            if not data:
                continue
            for key in ['open', 'high', 'low', 'volume']:
                if key in data:
                    talib_data[key] = data[key][0].values
            talib_data['close'] = data['price'][0].values
            expected_result = talib_fn(talib_data, **t.call_kwargs)[-1]
            np.testing.assert_allclose(talib_result, expected_result)

    def test_talib_with_minute_data(self):

        ma_one_day_minutes = ta.MA(timeperiod=10, bars='minute')

        # Assert that the BatchTransform window length is enough to cover
        # the amount of minutes in the timeperiod.

        # Here, 10 minutes only needs a window length of 1.
        self.assertEquals(1, ma_one_day_minutes.window_length)

        # With minutes greater than the 390, i.e. one trading day, we should
        # have a window_length of two days.
        ma_two_day_minutes = ta.MA(timeperiod=490, bars='minute')
        self.assertEquals(2, ma_two_day_minutes.window_length)

        # TODO: Ensure that the lookback into the datapanel is returning
        # expected results.
        # Requires supplying minute instead of day data to the unit test.
        # When adding test data, should add more minute events than the
        # timeperiod to ensure that lookback is behaving properly.
<commit_msg>TST: Disable talib default params test.<commit_after>#
# Copyright 2013 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import pytz
import numpy as np
import pandas as pd

from datetime import timedelta, datetime
from unittest import TestCase, skip

from six.moves import range

from zipline.utils.test_utils import setup_logger

from zipline.protocol import Event
from zipline.sources import SpecificEquityTrades
from zipline.transforms.utils import StatefulTransform, EventWindow
from zipline.transforms import MovingVWAP
from zipline.transforms import MovingAverage
from zipline.transforms import MovingStandardDev
from zipline.transforms import Returns
import zipline.utils.factory as factory

from zipline.test_algorithms import TALIBAlgorithm

def to_dt(msg):
    return Event({'dt': msg})

class NoopEventWindow(EventWindow):
    """
    A no-op EventWindow subclass for testing the base EventWindow logic.
    Keeps lists of all added and dropped events.
    """
    def __init__(self, market_aware, days, delta):
        EventWindow.__init__(self, market_aware, days, delta)

        self.added = []
        self.removed = []

    def handle_add(self, event):
        self.added.append(event)

    def handle_remove(self, event):
        self.removed.append(event)

class TestEventWindow(TestCase):
    def setUp(self):
        self.sim_params = factory.create_simulation_parameters()

        setup_logger(self)

        self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc)
        self.eleven_normal_days = [self.monday + i * timedelta(days=1)
                                   for i in range(11)]

        # Modify the end of the period slightly to exercise the
        # incomplete day logic.
        self.eleven_normal_days[-1] -= timedelta(minutes=1)
        self.eleven_normal_days.append(self.monday +
                                       timedelta(days=11, seconds=1))

        # Second set of dates to test holiday handling.
        self.jul4_monday = datetime(2012, 7, 2, 16, tzinfo=pytz.utc)
        self.week_of_jul4 = [self.jul4_monday + i * timedelta(days=1)
                             for i in range(5)]

    def test_market_aware_window_normal_week(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=3
        )
        events = [to_dt(date) for date in self.eleven_normal_days]
        lengths = []
        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        # The window stretches out during the weekend because we wait
        # to drop events until the weekend ends. The last window is
        # briefly longer because it doesn't complete a full day.  The
        # window then shrinks once the day completes
        self.assertEquals(lengths, [1, 2, 3, 3, 3, 4, 5, 5, 5, 3, 4, 3])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-3])

    def test_market_aware_window_holiday(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=2
        )
        events = [to_dt(date) for date in self.week_of_jul4]
        lengths = []

        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        self.assertEquals(lengths, [1, 2, 3, 3, 2])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-2])

    def tearDown(self):
        setup_logger(self)

class TestFinanceTransforms(TestCase):

    def setUp(self):
        self.sim_params = factory.create_simulation_parameters()
        setup_logger(self)

        trade_history = factory.create_trade_history(
            133,
            [10.0, 10.0, 11.0, 11.0],
            [100, 100, 100, 300],
            timedelta(days=1),
            self.sim_params
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

    def tearDown(self):
        self.log_handler.pop_application()

    def test_vwap(self):
        vwap = MovingVWAP(
            market_aware=True,
            window_length=2
        )
        transformed = list(vwap.transform(self.source))

        # Output values
        tnfm_vals = [message[vwap.get_hash()] for message in transformed]
        # "Hand calculated" values.
        expected = [
            (10.0 * 100) / 100.0,
            ((10.0 * 100) + (10.0 * 100)) / (200.0),
            # We should drop the first event here.
            ((10.0 * 100) + (11.0 * 100)) / (200.0),
            # We should drop the second event here.
            ((11.0 * 100) + (11.0 * 300)) / (400.0)
        ]

        # Output should match the expected.
        self.assertEquals(tnfm_vals, expected)

    def test_returns(self):
        # Daily returns.
        returns = Returns(1)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        # No returns for the first event because we don't have a
        # previous close.
        expected = [0.0, 0.0, 0.1, 0.0]

        self.assertEquals(tnfm_vals, expected)

        # Two-day returns.  An extra kink here is that the
        # factory will automatically skip a weekend for the
        # last event. Results shouldn't notice this blip.

        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0, 13.0],
            [100, 100, 100, 300, 100],
            timedelta(days=1),
            self.sim_params
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        returns = StatefulTransform(Returns, 2)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        expected = [
            0.0,
            0.0,
            (13.0 - 10.0) / 10.0,
            (12.0 - 15.0) / 15.0,
            (13.0 - 13.0) / 13.0
        ]

        self.assertEquals(tnfm_vals, expected)

    def test_moving_average(self):

        mavg = MovingAverage(
            market_aware=True,
            fields=['price', 'volume'],
            window_length=2
        )

        transformed = list(mavg.transform(self.source))
        # Output values.
        tnfm_prices = [message[mavg.get_hash()].price
                       for message in transformed]
        tnfm_volumes = [message[mavg.get_hash()].volume
                        for message in transformed]

        # "Hand-calculated" values
        expected_prices = [
            ((10.0) / 1.0),
            ((10.0 + 10.0) / 2.0),
            # First event should get dropped here.
            ((10.0 + 11.0) / 2.0),
            # Second event should get dropped here.
            ((11.0 + 11.0) / 2.0)
        ]
        expected_volumes = [
            ((100.0) / 1.0),
            ((100.0 + 100.0) / 2.0),
            # First event should get dropped here.
            ((100.0 + 100.0) / 2.0),
            # Second event should get dropped here.
            ((100.0 + 300.0) / 2.0)
        ]

        self.assertEquals(tnfm_prices, expected_prices)
        self.assertEquals(tnfm_volumes, expected_volumes)

    def test_moving_stddev(self):
        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0],
            [100, 100, 100, 100],
            timedelta(days=1),
            self.sim_params
        )

        stddev = MovingStandardDev(
            market_aware=True,
            window_length=3,
        )

        self.source = SpecificEquityTrades(event_list=trade_history)

        transformed = list(stddev.transform(self.source))

        vals = [message[stddev.get_hash()] for message in transformed]

        expected = [
            None,
            np.std([10.0, 15.0], ddof=1),
            np.std([10.0, 15.0, 13.0], ddof=1),
            np.std([15.0, 13.0, 12.0], ddof=1),
        ]

        # np has odd rounding behavior, cf.
        # http://docs.scipy.org/doc/np/reference/generated/np.std.html
        for v1, v2 in zip(vals, expected):

            if v1 is None:
                self.assertIsNone(v2)
                continue
            self.assertEquals(round(v1, 5), round(v2, 5))

############################################################
# Test TALIB

import talib
import zipline.transforms.ta as ta

class TestTALIB(TestCase):
    def setUp(self):
        setup_logger(self)
        sim_params = factory.create_simulation_parameters(
            start=datetime(1990, 1, 1, tzinfo=pytz.utc),
            end=datetime(1990, 3, 30, tzinfo=pytz.utc))
        self.source, self.panel = \
            factory.create_test_panel_ohlc_source(sim_params)

    @skip
    def test_talib_with_default_params(self):
        BLACKLIST = ['make_transform', 'BatchTransform',
                     # TODO: Figure out why MAVP generates a KeyError
                     'MAVP']
        names = [n for n in dir(ta) if n[0].isupper()
                 and n not in BLACKLIST]

        for name in names:
            print(name)
            zipline_transform = getattr(ta, name)(sid=0)
            talib_fn = getattr(talib.abstract, name)

            start = datetime(1990, 1, 1, tzinfo=pytz.utc)
            end = start + timedelta(days=zipline_transform.lookback + 10)
            sim_params = factory.create_simulation_parameters(
                start=start, end=end)
            source, panel = \
                factory.create_test_panel_ohlc_source(sim_params)

            algo = TALIBAlgorithm(talib=zipline_transform)
            algo.run(source)

            zipline_result = np.array(
                algo.talib_results[zipline_transform][-1])

            talib_data = dict()
            data = zipline_transform.window
            # TODO: Figure out if we are clobbering the tests by this
            # protection against empty windows
            if not data:
                continue
            for key in ['open', 'high', 'low', 'volume']:
                if key in data:
                    talib_data[key] = data[key][0].values
            talib_data['close'] = data['price'][0].values
            expected_result = talib_fn(talib_data)

            if isinstance(expected_result, list):
                expected_result = np.array([e[-1] for e in expected_result])
            else:
                expected_result = np.array(expected_result[-1])
            if not (np.all(np.isnan(zipline_result))
                    and np.all(np.isnan(expected_result))):
                self.assertTrue(np.allclose(zipline_result, expected_result))
            else:
                print('--- NAN')

            # reset generator so next iteration has data
            # self.source, self.panel = \
                # factory.create_test_panel_ohlc_source(self.sim_params)

    def test_multiple_talib_with_args(self):
        zipline_transforms = [ta.MA(timeperiod=10),
                              ta.MA(timeperiod=25)]
        talib_fn = talib.abstract.MA
        algo = TALIBAlgorithm(talib=zipline_transforms)
        algo.run(self.source)
        # Test if computed values match those computed by pandas rolling mean.
        sid = 0
        talib_values = np.array([x[sid] for x in
                                 algo.talib_results[zipline_transforms[0]]])
        np.testing.assert_array_equal(talib_values,
                                      pd.rolling_mean(self.panel[0]['price'],
                                                      10).values)
        talib_values = np.array([x[sid] for x in
                                 algo.talib_results[zipline_transforms[1]]])
        np.testing.assert_array_equal(talib_values,
                                      pd.rolling_mean(self.panel[0]['price'],
                                                      25).values)
        for t in zipline_transforms:
            talib_result = np.array(algo.talib_results[t][-1])
            talib_data = dict()
            data = t.window
            # TODO: Figure out if we are clobbering the tests by this
            # protection against empty windows
            if not data:
                continue
            for key in ['open', 'high', 'low', 'volume']:
                if key in data:
                    talib_data[key] = data[key][0].values
            talib_data['close'] = data['price'][0].values
            expected_result = talib_fn(talib_data, **t.call_kwargs)[-1]
            np.testing.assert_allclose(talib_result, expected_result)

    def test_talib_with_minute_data(self):

        ma_one_day_minutes = ta.MA(timeperiod=10, bars='minute')

        # Assert that the BatchTransform window length is enough to cover
        # the amount of minutes in the timeperiod.

        # Here, 10 minutes only needs a window length of 1.
        self.assertEquals(1, ma_one_day_minutes.window_length)

        # With minutes greater than the 390, i.e. one trading day, we should
        # have a window_length of two days.
        ma_two_day_minutes = ta.MA(timeperiod=490, bars='minute')
        self.assertEquals(2, ma_two_day_minutes.window_length)

        # TODO: Ensure that the lookback into the datapanel is returning
        # expected results.
        # Requires supplying minute instead of day data to the unit test.
        # When adding test data, should add more minute events than the
        # timeperiod to ensure that lookback is behaving properly.
<|endoftext|>