<commit_before>import io
import nose.tools as nt
from nose.tools import nottest
from converters import (
    ConverterLaTeX, ConverterMarkdown, ConverterPy, ConverterHTML
)

def test_evens():
    ######
    # for now, we don't need to really run inkscape to extract svg
    # from file, on unix, for test, we monkeypathc it to 'true'
    # which does not fail as doing anything.
    ####
    ConverterLaTeX.inkscape = 'true'

    # commenting rst for now as travis build
    # fail because of pandoc version.
    converters = [
                 #(ConverterRST, 'rst'),
                  (ConverterMarkdown, 'md'),
                  (ConverterLaTeX, 'tex'),
                  (ConverterPy, 'py'),
                  (ConverterHTML, 'html')
                ]
    reflist = [
            'tests/ipynbref/IntroNumPy.orig'
            ]
    for root in reflist:
        for conv, ext in converters:
            yield test_conversion, conv, root + '.ipynb', root + '.' + ext

@nottest
def compfiles(stra, strb):
    nt.assert_equal(map(unicode.strip, stra.split('\n')),
                    map(unicode.strip, strb.split('\n')))

@nottest
def test_conversion(ConverterClass, ipynb, ref_file):
    converter = ConverterClass(ipynb)
    converter.read()
    cv = converter.convert()
    with io.open(ref_file) as ref:
        value = ref.read()
        compfiles(cv, value)
<commit_msg>don't compare empty lines (stupid pandoc)<commit_after>import io
import nose.tools as nt
from nose.tools import nottest
from converters import (
    ConverterLaTeX, ConverterMarkdown, ConverterPy, ConverterHTML
)

@nottest
def cleanfile(stn):
    return filter(None, map(unicode.strip, stn.split('\n')))

def test_evens():
    ######
    # for now, we don't need to really run inkscape to extract svg
    # from file, on unix, for test, we monkeypathc it to 'true'
    # which does not fail as doing anything.
    ####
    ConverterLaTeX.inkscape = 'true'

    # commenting rst for now as travis build
    # fail because of pandoc version.
    converters = [
                 #(ConverterRST, 'rst'),
                  (ConverterMarkdown, 'md'),
                  (ConverterLaTeX, 'tex'),
                  (ConverterPy, 'py'),
                  (ConverterHTML, 'html')
                ]
    reflist = [
            'tests/ipynbref/IntroNumPy.orig'
            ]
    for root in reflist:
        for conv, ext in converters:
            yield test_conversion, conv, root + '.ipynb', root + '.' + ext

@nottest
def compfiles(stra, strb):
    nt.assert_equal(cleanfile(stra),
                    cleanfile(strb))

@nottest
def test_conversion(ConverterClass, ipynb, ref_file):
    converter = ConverterClass(ipynb)
    converter.read()
    cv = converter.convert()
    with io.open(ref_file) as ref:
        value = ref.read()
        compfiles(cv, value)
<|endoftext|>


<commit_before>import pytest
import os

import pandas as pd
from pandas.util.testing import assert_frame_equal

from aneris import cli

# decorator for slow-running tests
slow = pytest.mark.skipif(
    not pytest.config.getoption("--runslow"),
    reason="need --runslow option to run"
)

# This is a class that runs all tests through the harmonize CLI Note that it
# uses the actual harmonize API rather than subprocessing the CLI because
# coveralls does not pick up the lines covered in CLI calls when in a docker
# container.
#
# I don't know why. I spent 4+ hours digging. I am done. I hope I never have to
# worry about this again.
class TestHarmonizeRegression():

    def _run(self, prefix, inf, checkf, hist='history.csv', reg='message.csv'):
        # path setup
        here = os.path.join(os.path.dirname(os.path.realpath(__file__)))
        prefix = os.path.join(here, prefix)
        add_prefix = lambda f: os.path.join(prefix, f)

        # get all arguments
        hist = add_prefix(hist)
        reg = add_prefix(reg)
        rc = add_prefix('aneris.yaml')
        inf = add_prefix(inf)
        outf = add_prefix('test_harmonized.xlsx')
        if os.path.exists(outf):
            os.remove(outf)

        # run
        print(inf, hist, reg, rc, prefix, 'test')
        cli.harmonize(inf, hist, reg, rc, prefix, 'test')

        # test
        xfile = os.path.join(prefix, checkf)
        x = pd.read_excel(xfile, sheetname='data')
        y = pd.read_excel(outf, sheetname='data')
        assert_frame_equal(x, y)

    def test_basic_run(self):
        prefix = 'test_data'
        inf = 'model.xls'
        checkf = 'test_basic_run.xlsx'
        self._run(prefix, inf, checkf, hist='history.xls', reg='regions.csv')

    @slow
    def test_message_ref(self):
        prefix = 'regression_data'
        inf = 'MESSAGE-GLOBIOM_SSP2-Ref-SPA0-V25_unharmonized.xlsx'
        checkf = 'test_regress_ssp2_ref.xlsx'
        self._run(prefix, inf, checkf)

    @slow
    def test_message_no_sheet(self):
        prefix = 'regression_data'
        inf = 'no_sheet.xlsx'
        checkf = 'test_regress_ssp2_no_sheet.xlsx'
        self._run(prefix, inf, checkf)

    @slow
    def test_message_empty_sheet(self):
        prefix = 'regression_data'
        inf = 'empty_sheet.xlsx'
        checkf = 'test_regress_ssp2_empty_sheet.xlsx'
        self._run(prefix, inf, checkf)
<commit_msg>add a tear down section to regression tests<commit_after>import pytest
import os

import pandas as pd
from pandas.util.testing import assert_frame_equal

from aneris import cli

# decorator for slow-running tests
slow = pytest.mark.skipif(
    not pytest.config.getoption("--runslow"),
    reason="need --runslow option to run"
)

# This is a class that runs all tests through the harmonize CLI Note that it
# uses the actual harmonize API rather than subprocessing the CLI because
# coveralls does not pick up the lines covered in CLI calls when in a docker
# container.
#
# I don't know why. I spent 4+ hours digging. I am done. I hope I never have to
# worry about this again.
class TestHarmonizeRegression():

    def _run(self, prefix, inf, checkf, hist='history.csv', reg='message.csv'):
        # path setup
        here = os.path.join(os.path.dirname(os.path.realpath(__file__)))
        prefix = os.path.join(here, prefix)
        add_prefix = lambda f: os.path.join(prefix, f)

        # get all arguments
        hist = add_prefix(hist)
        reg = add_prefix(reg)
        rc = add_prefix('aneris.yaml')
        inf = add_prefix(inf)
        outf = add_prefix('test_harmonized.xlsx')
        if os.path.exists(outf):
            os.remove(outf)

        # run
        print(inf, hist, reg, rc, prefix, 'test')
        cli.harmonize(inf, hist, reg, rc, prefix, 'test')

        # test
        xfile = os.path.join(prefix, checkf)
        x = pd.read_excel(xfile, sheetname='data')
        y = pd.read_excel(outf, sheetname='data')
        assert_frame_equal(x, y)

        clean = [outf, add_prefix('test_metadata.xlsx')]
        for f in clean:
            os.remove(f)

    def test_basic_run(self):
        prefix = 'test_data'
        inf = 'model.xls'
        checkf = 'test_basic_run.xlsx'
        self._run(prefix, inf, checkf, hist='history.xls', reg='regions.csv')

    @slow
    def test_message_ref(self):
        prefix = 'regression_data'
        inf = 'MESSAGE-GLOBIOM_SSP2-Ref-SPA0-V25_unharmonized.xlsx'
        checkf = 'test_regress_ssp2_ref.xlsx'
        self._run(prefix, inf, checkf)

    @slow
    def test_message_no_sheet(self):
        prefix = 'regression_data'
        inf = 'no_sheet.xlsx'
        checkf = 'test_regress_ssp2_no_sheet.xlsx'
        self._run(prefix, inf, checkf)

    @slow
    def test_message_empty_sheet(self):
        prefix = 'regression_data'
        inf = 'empty_sheet.xlsx'
        checkf = 'test_regress_ssp2_empty_sheet.xlsx'
        self._run(prefix, inf, checkf)
<|endoftext|>


<commit_before>#!/usr/bin/env python

# This file is part of parallel-ssh.

# Copyright (C) 2015 Panos Kittenis

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, version 2.1.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""Unittests for :mod:`pssh.SSHClient` class"""

import gevent
import socket
import time
import unittest
from pssh import SSHClient, ParallelSSHClient, UnknownHostException, AuthenticationException,\
     logger, ConnectionErrorException, UnknownHostException, SSHException
from embedded_server.embedded_server import start_server, make_socket, logger as server_logger, \
     paramiko_logger
from embedded_server.fake_agent import FakeAgent
import paramiko
import os
from test_pssh_client import USER_KEY
import random, string

USER_KEY = paramiko.RSAKey.from_private_key_file(
    os.path.sep.join([os.path.dirname(__file__), 'test_client_private_key']))

class SSHClientTest(unittest.TestCase):

    def setUp(self):
        self.fake_cmd = 'echo me'
        self.fake_resp = 'me'
        self.user_key = USER_KEY
        self.host = '127.0.0.1'
        self.listen_socket = make_socket(self.host)
        self.listen_port = self.listen_socket.getsockname()[1]
        self.server = start_server(self.listen_socket)

    def tearDown(self):
        del self.server
        del self.listen_socket

    def test_ssh_client_sftp(self):
        """Test SFTP features of SSHClient. Copy local filename to server,
        check that data in both files is the same, make new directory on
        server, remove files and directory."""
        test_file_data = 'test'
        local_filename = 'test_file'
        remote_test_dir, remote_filename = 'remote_test_dir', 'test_file_copy'
        remote_filename = os.path.sep.join([remote_test_dir, remote_filename])
        remote_dir = 'remote_dir'
        test_file = open(local_filename, 'w')
        test_file.writelines([test_file_data + os.linesep])
        test_file.close()
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        client.copy_file(local_filename, remote_filename)
        self.assertTrue(os.path.isdir(remote_test_dir),
                        msg="SFTP create remote directory failed")
        self.assertTrue(os.path.isfile(remote_filename),
                        msg="SFTP copy failed")
        copied_file = open(remote_filename, 'r')
        copied_file_data = copied_file.readlines()[0].strip()
        copied_file.close()
        self.assertEqual(test_file_data, copied_file_data,
                         msg="Data in destination file %s does \
not match source %s" % (copied_file_data, test_file_data))
        for filepath in [local_filename, remote_filename]:
            os.unlink(filepath)
        client.mkdir(client._make_sftp(), remote_dir)
        self.assertTrue(os.path.isdir(remote_dir))
        for dirpath in [remote_dir, remote_test_dir]:
            os.rmdir(dirpath)
        del client

    def test_ssh_client_directory(self):
        """Tests copying directories with SSH client. Copy all the files from
        local directory to server, then make sure they are all present."""
        test_file_data = 'test'
        local_test_path = 'directory_test'
        remote_test_path = 'directory_test_copied'
        os.mkdir(local_test_path)
        remote_file_paths = []
        for i in range(0, 10):
            local_file_path = os.path.join(local_test_path, 'foo' + str(i))
            remote_file_path = os.path.join(remote_test_path, 'foo' + str(i))
            remote_file_paths.append(remote_file_path)
            test_file = open(local_file_path, 'w')
            test_file.write(test_file_data)
            test_file.close()
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        client.copy_file(local_test_path, remote_test_path)
        for path in remote_file_paths:
            self.assertTrue(os.path.isfile(path))

    def test_ssh_agent_authentication(self):
        """Test authentication via SSH agent.
        Do not provide public key to use when creating SSHClient,
        instead override the client's agent with our own fake SSH agent,
        add our to key to agent and try to login to server.
        Key should be automatically picked up from the overriden agent"""
        agent = FakeAgent()
        agent.add_key(USER_KEY)
        client = SSHClient(self.host, port=self.listen_port,
                           _agent=agent)
        channel, host, stdout, stderr = client.exec_command(self.fake_cmd)
        channel.close()
        output = list(stdout)
        stderr = list(stderr)
        expected = [self.fake_resp]
        self.assertEqual(expected, output,
                         msg = "Got unexpected command output - %s" % (output,))
        del client

    def test_ssh_client_conn_failure(self):
        """Test connection error failure case - ConnectionErrorException"""
        self.assertRaises(ConnectionErrorException,
                          SSHClient, '127.0.0.100', port=self.listen_port,
                          pkey=self.user_key, num_retries=0)

    def test_ssh_client_retries(self):
        """Test connection error exceptions"""
        self.assertRaises(ConnectionErrorException, SSHClient, '127.0.0.100', port=self.listen_port,
                          pkey=self.user_key, num_retries=1)
        host = ''.join([random.choice(string.ascii_letters) for n in xrange(8)])
        self.assertRaises(UnknownHostException, SSHClient, host, port=self.listen_port,
                          pkey=self.user_key, num_retries=1)

    def test_ssh_client_unknown_host_failure(self):
        """Test connection error failure case - ConnectionErrorException"""
        host = ''.join([random.choice(string.ascii_letters) for n in xrange(8)])
        self.assertRaises(UnknownHostException,
                          SSHClient, host, port=self.listen_port,
                          pkey=self.user_key, num_retries=0)

    def test_ssh_client_pty(self):
        """Test that we get a new pty for our non-interactive SSH sessions"""
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        channel = client.client.get_transport().open_session()
        self.assertFalse(channel.event.is_set(),
                         msg="Got pty without requesting it")
        channel.get_pty()
        self.assertTrue(channel.event.is_set(),
                        msg="Requested pty but got none")
        channel.close()
        del channel
        del client

if __name__ == '__main__':
    unittest.main()
<commit_msg>Forgot to have test_ssh_client_directory clean up after itself.<commit_after>#!/usr/bin/env python

# This file is part of parallel-ssh.

# Copyright (C) 2015 Panos Kittenis

# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation, version 2.1.

# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.

# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA

"""Unittests for :mod:`pssh.SSHClient` class"""

import gevent
import socket
import time
import shutil
import unittest
from pssh import SSHClient, ParallelSSHClient, UnknownHostException, AuthenticationException,\
     logger, ConnectionErrorException, UnknownHostException, SSHException
from embedded_server.embedded_server import start_server, make_socket, logger as server_logger, \
     paramiko_logger
from embedded_server.fake_agent import FakeAgent
import paramiko
import os
from test_pssh_client import USER_KEY
import random, string

USER_KEY = paramiko.RSAKey.from_private_key_file(
    os.path.sep.join([os.path.dirname(__file__), 'test_client_private_key']))

class SSHClientTest(unittest.TestCase):

    def setUp(self):
        self.fake_cmd = 'echo me'
        self.fake_resp = 'me'
        self.user_key = USER_KEY
        self.host = '127.0.0.1'
        self.listen_socket = make_socket(self.host)
        self.listen_port = self.listen_socket.getsockname()[1]
        self.server = start_server(self.listen_socket)

    def tearDown(self):
        del self.server
        del self.listen_socket

    def test_ssh_client_sftp(self):
        """Test SFTP features of SSHClient. Copy local filename to server,
        check that data in both files is the same, make new directory on
        server, remove files and directory."""
        test_file_data = 'test'
        local_filename = 'test_file'
        remote_test_dir, remote_filename = 'remote_test_dir', 'test_file_copy'
        remote_filename = os.path.sep.join([remote_test_dir, remote_filename])
        remote_dir = 'remote_dir'
        test_file = open(local_filename, 'w')
        test_file.writelines([test_file_data + os.linesep])
        test_file.close()
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        client.copy_file(local_filename, remote_filename)
        self.assertTrue(os.path.isdir(remote_test_dir),
                        msg="SFTP create remote directory failed")
        self.assertTrue(os.path.isfile(remote_filename),
                        msg="SFTP copy failed")
        copied_file = open(remote_filename, 'r')
        copied_file_data = copied_file.readlines()[0].strip()
        copied_file.close()
        self.assertEqual(test_file_data, copied_file_data,
                         msg="Data in destination file %s does \
not match source %s" % (copied_file_data, test_file_data))
        for filepath in [local_filename, remote_filename]:
            os.unlink(filepath)
        client.mkdir(client._make_sftp(), remote_dir)
        self.assertTrue(os.path.isdir(remote_dir))
        for dirpath in [remote_dir, remote_test_dir]:
            os.rmdir(dirpath)
        del client

    def test_ssh_client_directory(self):
        """Tests copying directories with SSH client. Copy all the files from
        local directory to server, then make sure they are all present."""
        test_file_data = 'test'
        local_test_path = 'directory_test'
        remote_test_path = 'directory_test_copied'
        os.mkdir(local_test_path)
        remote_file_paths = []
        for i in range(0, 10):
            local_file_path = os.path.join(local_test_path, 'foo' + str(i))
            remote_file_path = os.path.join(remote_test_path, 'foo' + str(i))
            remote_file_paths.append(remote_file_path)
            test_file = open(local_file_path, 'w')
            test_file.write(test_file_data)
            test_file.close()
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        client.copy_file(local_test_path, remote_test_path)
        for path in remote_file_paths:
            self.assertTrue(os.path.isfile(path))
        shutil.rmtree(local_test_path)
        shutil.rmtree(remote_test_path)

    def test_ssh_agent_authentication(self):
        """Test authentication via SSH agent.
        Do not provide public key to use when creating SSHClient,
        instead override the client's agent with our own fake SSH agent,
        add our to key to agent and try to login to server.
        Key should be automatically picked up from the overriden agent"""
        agent = FakeAgent()
        agent.add_key(USER_KEY)
        client = SSHClient(self.host, port=self.listen_port,
                           _agent=agent)
        channel, host, stdout, stderr = client.exec_command(self.fake_cmd)
        channel.close()
        output = list(stdout)
        stderr = list(stderr)
        expected = [self.fake_resp]
        self.assertEqual(expected, output,
                         msg = "Got unexpected command output - %s" % (output,))
        del client

    def test_ssh_client_conn_failure(self):
        """Test connection error failure case - ConnectionErrorException"""
        self.assertRaises(ConnectionErrorException,
                          SSHClient, '127.0.0.100', port=self.listen_port,
                          pkey=self.user_key, num_retries=0)

    def test_ssh_client_retries(self):
        """Test connection error exceptions"""
        self.assertRaises(ConnectionErrorException, SSHClient, '127.0.0.100', port=self.listen_port,
                          pkey=self.user_key, num_retries=1)
        host = ''.join([random.choice(string.ascii_letters) for n in xrange(8)])
        self.assertRaises(UnknownHostException, SSHClient, host, port=self.listen_port,
                          pkey=self.user_key, num_retries=1)

    def test_ssh_client_unknown_host_failure(self):
        """Test connection error failure case - ConnectionErrorException"""
        host = ''.join([random.choice(string.ascii_letters) for n in xrange(8)])
        self.assertRaises(UnknownHostException,
                          SSHClient, host, port=self.listen_port,
                          pkey=self.user_key, num_retries=0)

    def test_ssh_client_pty(self):
        """Test that we get a new pty for our non-interactive SSH sessions"""
        client = SSHClient(self.host, port=self.listen_port,
                           pkey=self.user_key)
        channel = client.client.get_transport().open_session()
        self.assertFalse(channel.event.is_set(),
                         msg="Got pty without requesting it")
        channel.get_pty()
        self.assertTrue(channel.event.is_set(),
                        msg="Requested pty but got none")
        channel.close()
        del channel
        del client

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

################################################################################
# Copyright (c) 2020 ContinualAI Research                                      #
# Copyrights licensed under the CC BY 4.0 License.                             #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 1-06-2020                                                              #
# Author(s): Andrea Cossu                                                #
# E-mail: contact@continualai.org                                              #
# Website: clair.continualai.org                                               #
################################################################################

import unittest

import torch

from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor, Compose
from torch.optim import SGD
from torch.nn import CrossEntropyLoss

from avalanche.extras.models import SimpleMLP
from avalanche.evaluation import EvalProtocol
from avalanche.evaluation.metrics import ACC
from avalanche.benchmarks.scenarios import DatasetPart
from avalanche.training.strategies import Naive, Cumulative, Replay, GDumb
from avalanche.benchmarks import NCBenchmark, NCStepInfo

#from avalanche.training.plugins import ReplayPlugin, GDumbPlugin

device = 'cpu'

class StrategyTest(unittest.TestCase):

    def test_naive(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        strategy = Naive(model, 'classifier', optimizer, criterion,
                evaluation_protocol=eval_protocol, train_mb_size=100,
                train_epochs=4, test_mb_size=100, device=device)

        self.run_strategy(nc_scenario, strategy)

    def test_replay(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        def reinit(m):
            with torch.no_grad():
                for p in m.parameters():
                    torch.nn.init.uniform_(p, -1, -1)

        strategy = Replay(model, 'classifier', optimizer, criterion,
                mem_size=200, reinit_model_before_step=True,
                reinit_function=reinit, # None to use default init function
                evaluation_protocol=eval_protocol,
                train_mb_size=100,
                train_epochs=4, test_mb_size=100, device=device, plugins=None
                )

        self.run_strategy(nc_scenario, strategy)

    def test_cumulative(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        strategy = Cumulative(model, 'classifier', optimizer, criterion,
                train_mb_size=100,
                evaluation_protocol=eval_protocol,
                train_epochs=4, test_mb_size=100, device=device)

        self.run_strategy(nc_scenario, strategy)

    def test_gdumb(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-2)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        def reinit(m):
            with torch.no_grad():
                for p in m.parameters():
                    torch.nn.init.uniform_(p, -1, 1)

        strategy = GDumb(model, 'classifier', optimizer, criterion,
                mem_size=2000, reinit_model_before_step=True,
                reinit_function=reinit, # None to use default init function
                train_mb_size=64,
                evaluation_protocol=eval_protocol,
                train_epochs=10, test_mb_size=100, device=device,
                plugins=None)

        self.run_strategy(nc_scenario, strategy)

    def load_dataset(self):

        mnist_train = MNIST('./data/mnist', train=True, download=True,
                transform=Compose([ToTensor()]))
        mnist_test = MNIST('./data/mnist', train=False, download=True,
                transform=Compose([ToTensor()]))
        return mnist_train, mnist_test

    def run_strategy(self, scenario, cl_strategy):

        print('Starting experiment...')
        results = []
        batch_info: NCStepInfo
        for batch_info in scenario:
            print("Start of step ", batch_info.current_step)

            cl_strategy.train(batch_info, num_workers=4)
            print('Training completed')

            print('Computing accuracy on the whole test set')
            results.append(cl_strategy.test(batch_info, DatasetPart.COMPLETE,
                                            num_workers=4))

if __name__ == '__main__':
    unittest.main()
<commit_msg>Removed unused import<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

################################################################################
# Copyright (c) 2020 ContinualAI Research                                      #
# Copyrights licensed under the CC BY 4.0 License.                             #
# See the accompanying LICENSE file for terms.                                 #
#                                                                              #
# Date: 1-06-2020                                                              #
# Author(s): Andrea Cossu                                                #
# E-mail: contact@continualai.org                                              #
# Website: clair.continualai.org                                               #
################################################################################

import unittest

import torch

from torchvision.datasets import MNIST
from torchvision.transforms import ToTensor, Compose
from torch.optim import SGD
from torch.nn import CrossEntropyLoss

from avalanche.extras.models import SimpleMLP
from avalanche.evaluation import EvalProtocol
from avalanche.evaluation.metrics import ACC
from avalanche.benchmarks.scenarios import DatasetPart
from avalanche.training.strategies import Naive, Cumulative, Replay, GDumb
from avalanche.benchmarks import NCBenchmark, NCStepInfo

device = 'cpu'

class StrategyTest(unittest.TestCase):

    def test_naive(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        strategy = Naive(model, 'classifier', optimizer, criterion,
                evaluation_protocol=eval_protocol, train_mb_size=100,
                train_epochs=4, test_mb_size=100, device=device)

        self.run_strategy(nc_scenario, strategy)

    def test_replay(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        def reinit(m):
            with torch.no_grad():
                for p in m.parameters():
                    torch.nn.init.uniform_(p, -1, -1)

        strategy = Replay(model, 'classifier', optimizer, criterion,
                mem_size=200, reinit_model_before_step=True,
                reinit_function=reinit, # None to use default init function
                evaluation_protocol=eval_protocol,
                train_mb_size=100,
                train_epochs=4, test_mb_size=100, device=device, plugins=None
                )

        self.run_strategy(nc_scenario, strategy)

    def test_cumulative(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-3)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False,
            shuffle=True, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        strategy = Cumulative(model, 'classifier', optimizer, criterion,
                train_mb_size=100,
                evaluation_protocol=eval_protocol,
                train_epochs=4, test_mb_size=100, device=device)

        self.run_strategy(nc_scenario, strategy)

    def test_gdumb(self):
        model = SimpleMLP()
        optimizer = SGD(model.parameters(), lr=1e-2)
        criterion = CrossEntropyLoss()
        mnist_train, mnist_test = self.load_dataset()
        nc_scenario = NCBenchmark(
            mnist_train, mnist_test, 5, task_labels=False, seed=1234)

        eval_protocol = EvalProtocol(
            metrics=[
                ACC(num_class=nc_scenario.n_classes)
            ])

        def reinit(m):
            with torch.no_grad():
                for p in m.parameters():
                    torch.nn.init.uniform_(p, -1, 1)

        strategy = GDumb(model, 'classifier', optimizer, criterion,
                mem_size=2000, reinit_model_before_step=True,
                reinit_function=reinit, # None to use default init function
                train_mb_size=64,
                evaluation_protocol=eval_protocol,
                train_epochs=10, test_mb_size=100, device=device,
                plugins=None)

        self.run_strategy(nc_scenario, strategy)

    def load_dataset(self):

        mnist_train = MNIST('./data/mnist', train=True, download=True,
                transform=Compose([ToTensor()]))
        mnist_test = MNIST('./data/mnist', train=False, download=True,
                transform=Compose([ToTensor()]))
        return mnist_train, mnist_test

    def run_strategy(self, scenario, cl_strategy):

        print('Starting experiment...')
        results = []
        batch_info: NCStepInfo
        for batch_info in scenario:
            print("Start of step ", batch_info.current_step)

            cl_strategy.train(batch_info, num_workers=4)
            print('Training completed')

            print('Computing accuracy on the whole test set')
            results.append(cl_strategy.test(batch_info, DatasetPart.COMPLETE,
                                            num_workers=4))

if __name__ == '__main__':
    unittest.main()
<|endoftext|>


<commit_before>#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import deque

import pytz
import numpy as np
import pandas as pd

from datetime import timedelta, datetime
from unittest import TestCase

from zipline import ndict

from zipline.utils.test_utils import setup_logger
from zipline.utils.date_utils import utcnow

from zipline.sources import SpecificEquityTrades
from zipline.transforms.utils import StatefulTransform, EventWindow
from zipline.transforms import MovingVWAP
from zipline.transforms import MovingAverage
from zipline.transforms import MovingStandardDev
from zipline.transforms import Returns
import zipline.utils.factory as factory

from zipline.test_algorithms import BatchTransformAlgorithm

def to_dt(msg):
    return ndict({'dt': msg})

class NoopEventWindow(EventWindow):
    """
    A no-op EventWindow subclass for testing the base EventWindow logic.
    Keeps lists of all added and dropped events.
    """
    def __init__(self, market_aware, days, delta):
        EventWindow.__init__(self, market_aware, days, delta)

        self.added = []
        self.removed = []

    def handle_add(self, event):
        self.added.append(event)

    def handle_remove(self, event):
        self.removed.append(event)

class TestEventWindow(TestCase):
    def setUp(self):
        setup_logger(self)

        self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc)
        self.eleven_normal_days = [self.monday + i * timedelta(days=1)
                                   for i in xrange(11)]

        # Modify the end of the period slightly to exercise the
        # incomplete day logic.
        self.eleven_normal_days[-1] -= timedelta(minutes=1)
        self.eleven_normal_days.append(self.monday +
                                       timedelta(days=11, seconds=1))

        # Second set of dates to test holiday handling.
        self.jul4_monday = datetime(2012, 7, 2, 16, tzinfo=pytz.utc)
        self.week_of_jul4 = [self.jul4_monday + i * timedelta(days=1)
                             for i in xrange(5)]

    def test_event_window_with_timedelta(self):

        # Keep all events within a 5 minute window.
        window = NoopEventWindow(
            market_aware=False,
            delta=timedelta(minutes=5),
            days=None
        )

        now = utcnow()

        # 15 dates, increasing in 1 minute increments.
        dates = [now + i * timedelta(minutes=1)
                 for i in xrange(15)]

        # Turn the dates into the format required by EventWindow.
        dt_messages = [to_dt(date) for date in dates]

        # Run all messages through the window and assert that we're adding
        # and removing messages appropriately. We start the enumeration at 1
        # for convenience.
        for num, message in enumerate(dt_messages, 1):
            window.update(message)

            # Assert that we've added the correct number of events.
            self.assertTrue(len(window.added), num)

            # Assert that we removed only events that fall outside (or
            # on the boundary of) the delta.
            for dropped in window.removed:
                self.assertTrue(
                    message.dt - dropped.dt >= timedelta(minutes=5))

    def test_market_aware_window_normal_week(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=3
        )
        events = [to_dt(date) for date in self.eleven_normal_days]
        lengths = []
        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        # The window stretches out during the weekend because we wait
        # to drop events until the weekend ends. The last window is
        # briefly longer because it doesn't complete a full day.  The
        # window then shrinks once the day completes
        self.assertEquals(lengths, [1, 2, 3, 3, 3, 4, 5, 5, 5, 3, 4, 3])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-3])

    def test_market_aware_window_holiday(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=2
        )
        events = [to_dt(date) for date in self.week_of_jul4]
        lengths = []

        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        self.assertEquals(lengths, [1, 2, 3, 3, 2])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-2])

    def tearDown(self):
        setup_logger(self)

class TestFinanceTransforms(TestCase):

    def setUp(self):
        self.trading_environment = factory.create_trading_environment()
        setup_logger(self)

        trade_history = factory.create_trade_history(
            133,
            [10.0, 10.0, 11.0, 11.0],
            [100, 100, 100, 300],
            timedelta(days=1),
            self.trading_environment
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

    def tearDown(self):
        self.log_handler.pop_application()

    def test_vwap(self):

        vwap = MovingVWAP(
            market_aware=False,
            delta=timedelta(days=2)
        )
        transformed = list(vwap.transform(self.source))

        # Output values
        tnfm_vals = [message[vwap.get_hash()] for message in transformed]
        # "Hand calculated" values.
        expected = [
            (10.0 * 100) / 100.0,
            ((10.0 * 100) + (10.0 * 100)) / (200.0),
            # We should drop the first event here.
            ((10.0 * 100) + (11.0 * 100)) / (200.0),
            # We should drop the second event here.
            ((11.0 * 100) + (11.0 * 300)) / (400.0)
        ]

        # Output should match the expected.
        self.assertEquals(tnfm_vals, expected)

    def test_returns(self):
        # Daily returns.
        returns = Returns(1)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        # No returns for the first event because we don't have a
        # previous close.
        expected = [0.0, 0.0, 0.1, 0.0]

        self.assertEquals(tnfm_vals, expected)

        # Two-day returns.  An extra kink here is that the
        # factory will automatically skip a weekend for the
        # last event. Results shouldn't notice this blip.

        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0, 13.0],
            [100, 100, 100, 300, 100],
            timedelta(days=1),
            self.trading_environment
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        returns = StatefulTransform(Returns, 2)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        expected = [
            0.0,
            0.0,
            (13.0 - 10.0) / 10.0,
            (12.0 - 15.0) / 15.0,
            (13.0 - 13.0) / 13.0
        ]

        self.assertEquals(tnfm_vals, expected)

    def test_moving_average(self):

        mavg = MovingAverage(
            market_aware=False,
            fields=['price', 'volume'],
            delta=timedelta(days=2),
        )

        transformed = list(mavg.transform(self.source))
        # Output values.
        tnfm_prices = [message[mavg.get_hash()].price
                       for message in transformed]
        tnfm_volumes = [message[mavg.get_hash()].volume
                        for message in transformed]

        # "Hand-calculated" values
        expected_prices = [
            ((10.0) / 1.0),
            ((10.0 + 10.0) / 2.0),
            # First event should get dropped here.
            ((10.0 + 11.0) / 2.0),
            # Second event should get dropped here.
            ((11.0 + 11.0) / 2.0)
        ]
        expected_volumes = [
            ((100.0) / 1.0),
            ((100.0 + 100.0) / 2.0),
            # First event should get dropped here.
            ((100.0 + 100.0) / 2.0),
            # Second event should get dropped here.
            ((100.0 + 300.0) / 2.0)
        ]

        self.assertEquals(tnfm_prices, expected_prices)
        self.assertEquals(tnfm_volumes, expected_volumes)

    def test_moving_stddev(self):
        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0],
            [100, 100, 100, 100],
            timedelta(hours=1),
            self.trading_environment
        )

        stddev = MovingStandardDev(
            market_aware=False,
            delta=timedelta(minutes=150),
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        transformed = list(stddev.transform(self.source))

        vals = [message[stddev.get_hash()] for message in transformed]

        expected = [
            None,
            np.std([10.0, 15.0], ddof=1),
            np.std([10.0, 15.0, 13.0], ddof=1),
            np.std([15.0, 13.0, 12.0], ddof=1),
        ]

        # np has odd rounding behavior, cf.
        # http://docs.scipy.org/doc/np/reference/generated/np.std.html
        for v1, v2 in zip(vals, expected):

            if v1 is None:
                self.assertIsNone(v2)
                continue
            self.assertEquals(round(v1, 5), round(v2, 5))

############################################################
# Test BatchTransform

class TestBatchTransform(TestCase):
    def setUp(self):
        setup_logger(self)
        self.source, self.df = factory.create_test_df_source()

    def test_event_window(self):
        algo = BatchTransformAlgorithm()
        algo.run(self.source)
        wl = algo.window_length
        # The following assertion depend on window length of 3
        self.assertEqual(wl, 3)
        self.assertEqual(algo.history_return_price_class[:wl],
                         [None] * wl,
                         "First three iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_class,))
        self.assertEqual(algo.history_return_price_decorator[:wl],
                         [None] * wl,
                         "First three iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_decorator,))
        # After three Nones, the next value should be a data frame
        self.assertTrue(isinstance(
            algo.history_return_price_class[wl],
            pd.DataFrame)
        )

        # Test whether arbitrary fields can be added to datapanel
        field = algo.history_return_arbitrary_fields[-1]
        self.assertTrue(
            'arbitrary' in field.items,
            'datapanel should contain column arbitrary'
        )

        self.assertTrue(all(
            field['arbitrary'].values.flatten() ==
            [123] * algo.window_length),
            'arbitrary dataframe should contain only "test"'
        )

        for data in algo.history_return_sid_filter[wl:]:
            self.assertIn(0, data.columns)
            self.assertNotIn(1, data.columns)

        for data in algo.history_return_field_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertNotIn('ignore', data.items)

        for data in algo.history_return_field_no_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertIn('ignore', data.items)

        for data in algo.history_return_ticks[wl:]:
            self.assertTrue(isinstance(data, deque))

        for data in algo.history_return_not_full:
            self.assertIsNot(data, None)

        # test overloaded class
        for test_history in [algo.history_return_price_class,
                             algo.history_return_price_decorator]:
            # starting at window length, the window should contain
            # consecutive (of window length) numbers up till the end.
            for i in range(algo.window_length, len(test_history)):
                np.testing.assert_array_equal(
                    range(i - algo.window_length + 1, i + 1),
                    test_history[i].values.flatten()
                )

    def test_passing_of_args(self):
        algo = BatchTransformAlgorithm(1, kwarg='str')
        self.assertEqual(algo.args, (1,))
        self.assertEqual(algo.kwargs, {'kwarg': 'str'})

        algo.run(self.source)
        expected_item = ((1, ), {'kwarg': 'str'})
        self.assertEqual(
            algo.history_return_args,
            [
                # 1990-01-03 - window not full
                None,
                # 1990-01-04 - window not full
                None,
                # 1990-01-05 - window not full, 3rd event
                None,
                # 1990-01-08 - window now full
                expected_item
            ])
<commit_msg>Fixes assert type of window length check.<commit_after>#
# Copyright 2012 Quantopian, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from collections import deque

import pytz
import numpy as np
import pandas as pd

from datetime import timedelta, datetime
from unittest import TestCase

from zipline import ndict

from zipline.utils.test_utils import setup_logger
from zipline.utils.date_utils import utcnow

from zipline.sources import SpecificEquityTrades
from zipline.transforms.utils import StatefulTransform, EventWindow
from zipline.transforms import MovingVWAP
from zipline.transforms import MovingAverage
from zipline.transforms import MovingStandardDev
from zipline.transforms import Returns
import zipline.utils.factory as factory

from zipline.test_algorithms import BatchTransformAlgorithm

def to_dt(msg):
    return ndict({'dt': msg})

class NoopEventWindow(EventWindow):
    """
    A no-op EventWindow subclass for testing the base EventWindow logic.
    Keeps lists of all added and dropped events.
    """
    def __init__(self, market_aware, days, delta):
        EventWindow.__init__(self, market_aware, days, delta)

        self.added = []
        self.removed = []

    def handle_add(self, event):
        self.added.append(event)

    def handle_remove(self, event):
        self.removed.append(event)

class TestEventWindow(TestCase):
    def setUp(self):
        setup_logger(self)

        self.monday = datetime(2012, 7, 9, 16, tzinfo=pytz.utc)
        self.eleven_normal_days = [self.monday + i * timedelta(days=1)
                                   for i in xrange(11)]

        # Modify the end of the period slightly to exercise the
        # incomplete day logic.
        self.eleven_normal_days[-1] -= timedelta(minutes=1)
        self.eleven_normal_days.append(self.monday +
                                       timedelta(days=11, seconds=1))

        # Second set of dates to test holiday handling.
        self.jul4_monday = datetime(2012, 7, 2, 16, tzinfo=pytz.utc)
        self.week_of_jul4 = [self.jul4_monday + i * timedelta(days=1)
                             for i in xrange(5)]

    def test_event_window_with_timedelta(self):

        # Keep all events within a 5 minute window.
        window = NoopEventWindow(
            market_aware=False,
            delta=timedelta(minutes=5),
            days=None
        )

        now = utcnow()

        # 15 dates, increasing in 1 minute increments.
        dates = [now + i * timedelta(minutes=1)
                 for i in xrange(15)]

        # Turn the dates into the format required by EventWindow.
        dt_messages = [to_dt(date) for date in dates]

        # Run all messages through the window and assert that we're adding
        # and removing messages appropriately. We start the enumeration at 1
        # for convenience.
        for num, message in enumerate(dt_messages, 1):
            window.update(message)

            # Assert that we've added the correct number of events.
            self.assertEquals(len(window.added), num)

            # Assert that we removed only events that fall outside (or
            # on the boundary of) the delta.
            for dropped in window.removed:
                self.assertTrue(
                    message.dt - dropped.dt >= timedelta(minutes=5))

    def test_market_aware_window_normal_week(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=3
        )
        events = [to_dt(date) for date in self.eleven_normal_days]
        lengths = []
        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        # The window stretches out during the weekend because we wait
        # to drop events until the weekend ends. The last window is
        # briefly longer because it doesn't complete a full day.  The
        # window then shrinks once the day completes
        self.assertEquals(lengths, [1, 2, 3, 3, 3, 4, 5, 5, 5, 3, 4, 3])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-3])

    def test_market_aware_window_holiday(self):
        window = NoopEventWindow(
            market_aware=True,
            delta=None,
            days=2
        )
        events = [to_dt(date) for date in self.week_of_jul4]
        lengths = []

        # Run the events.
        for event in events:
            window.update(event)
            # Record the length of the window after each event.
            lengths.append(len(window.ticks))

        self.assertEquals(lengths, [1, 2, 3, 3, 2])
        self.assertEquals(window.added, events)
        self.assertEquals(window.removed, events[:-2])

    def tearDown(self):
        setup_logger(self)

class TestFinanceTransforms(TestCase):

    def setUp(self):
        self.trading_environment = factory.create_trading_environment()
        setup_logger(self)

        trade_history = factory.create_trade_history(
            133,
            [10.0, 10.0, 11.0, 11.0],
            [100, 100, 100, 300],
            timedelta(days=1),
            self.trading_environment
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

    def tearDown(self):
        self.log_handler.pop_application()

    def test_vwap(self):

        vwap = MovingVWAP(
            market_aware=False,
            delta=timedelta(days=2)
        )
        transformed = list(vwap.transform(self.source))

        # Output values
        tnfm_vals = [message[vwap.get_hash()] for message in transformed]
        # "Hand calculated" values.
        expected = [
            (10.0 * 100) / 100.0,
            ((10.0 * 100) + (10.0 * 100)) / (200.0),
            # We should drop the first event here.
            ((10.0 * 100) + (11.0 * 100)) / (200.0),
            # We should drop the second event here.
            ((11.0 * 100) + (11.0 * 300)) / (400.0)
        ]

        # Output should match the expected.
        self.assertEquals(tnfm_vals, expected)

    def test_returns(self):
        # Daily returns.
        returns = Returns(1)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        # No returns for the first event because we don't have a
        # previous close.
        expected = [0.0, 0.0, 0.1, 0.0]

        self.assertEquals(tnfm_vals, expected)

        # Two-day returns.  An extra kink here is that the
        # factory will automatically skip a weekend for the
        # last event. Results shouldn't notice this blip.

        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0, 13.0],
            [100, 100, 100, 300, 100],
            timedelta(days=1),
            self.trading_environment
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        returns = StatefulTransform(Returns, 2)

        transformed = list(returns.transform(self.source))
        tnfm_vals = [message[returns.get_hash()] for message in transformed]

        expected = [
            0.0,
            0.0,
            (13.0 - 10.0) / 10.0,
            (12.0 - 15.0) / 15.0,
            (13.0 - 13.0) / 13.0
        ]

        self.assertEquals(tnfm_vals, expected)

    def test_moving_average(self):

        mavg = MovingAverage(
            market_aware=False,
            fields=['price', 'volume'],
            delta=timedelta(days=2),
        )

        transformed = list(mavg.transform(self.source))
        # Output values.
        tnfm_prices = [message[mavg.get_hash()].price
                       for message in transformed]
        tnfm_volumes = [message[mavg.get_hash()].volume
                        for message in transformed]

        # "Hand-calculated" values
        expected_prices = [
            ((10.0) / 1.0),
            ((10.0 + 10.0) / 2.0),
            # First event should get dropped here.
            ((10.0 + 11.0) / 2.0),
            # Second event should get dropped here.
            ((11.0 + 11.0) / 2.0)
        ]
        expected_volumes = [
            ((100.0) / 1.0),
            ((100.0 + 100.0) / 2.0),
            # First event should get dropped here.
            ((100.0 + 100.0) / 2.0),
            # Second event should get dropped here.
            ((100.0 + 300.0) / 2.0)
        ]

        self.assertEquals(tnfm_prices, expected_prices)
        self.assertEquals(tnfm_volumes, expected_volumes)

    def test_moving_stddev(self):
        trade_history = factory.create_trade_history(
            133,
            [10.0, 15.0, 13.0, 12.0],
            [100, 100, 100, 100],
            timedelta(hours=1),
            self.trading_environment
        )

        stddev = MovingStandardDev(
            market_aware=False,
            delta=timedelta(minutes=150),
        )
        self.source = SpecificEquityTrades(event_list=trade_history)

        transformed = list(stddev.transform(self.source))

        vals = [message[stddev.get_hash()] for message in transformed]

        expected = [
            None,
            np.std([10.0, 15.0], ddof=1),
            np.std([10.0, 15.0, 13.0], ddof=1),
            np.std([15.0, 13.0, 12.0], ddof=1),
        ]

        # np has odd rounding behavior, cf.
        # http://docs.scipy.org/doc/np/reference/generated/np.std.html
        for v1, v2 in zip(vals, expected):

            if v1 is None:
                self.assertIsNone(v2)
                continue
            self.assertEquals(round(v1, 5), round(v2, 5))

############################################################
# Test BatchTransform

class TestBatchTransform(TestCase):
    def setUp(self):
        setup_logger(self)
        self.source, self.df = factory.create_test_df_source()

    def test_event_window(self):
        algo = BatchTransformAlgorithm()
        algo.run(self.source)
        wl = algo.window_length
        # The following assertion depend on window length of 3
        self.assertEqual(wl, 3)
        self.assertEqual(algo.history_return_price_class[:wl],
                         [None] * wl,
                         "First three iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_class,))
        self.assertEqual(algo.history_return_price_decorator[:wl],
                         [None] * wl,
                         "First three iterations should return None." + "\n" +
                         "i.e. no returned values until window is full'" +
                         "%s" % (algo.history_return_price_decorator,))
        # After three Nones, the next value should be a data frame
        self.assertTrue(isinstance(
            algo.history_return_price_class[wl],
            pd.DataFrame)
        )

        # Test whether arbitrary fields can be added to datapanel
        field = algo.history_return_arbitrary_fields[-1]
        self.assertTrue(
            'arbitrary' in field.items,
            'datapanel should contain column arbitrary'
        )

        self.assertTrue(all(
            field['arbitrary'].values.flatten() ==
            [123] * algo.window_length),
            'arbitrary dataframe should contain only "test"'
        )

        for data in algo.history_return_sid_filter[wl:]:
            self.assertIn(0, data.columns)
            self.assertNotIn(1, data.columns)

        for data in algo.history_return_field_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertNotIn('ignore', data.items)

        for data in algo.history_return_field_no_filter[wl:]:
            self.assertIn('price', data.items)
            self.assertIn('ignore', data.items)

        for data in algo.history_return_ticks[wl:]:
            self.assertTrue(isinstance(data, deque))

        for data in algo.history_return_not_full:
            self.assertIsNot(data, None)

        # test overloaded class
        for test_history in [algo.history_return_price_class,
                             algo.history_return_price_decorator]:
            # starting at window length, the window should contain
            # consecutive (of window length) numbers up till the end.
            for i in range(algo.window_length, len(test_history)):
                np.testing.assert_array_equal(
                    range(i - algo.window_length + 1, i + 1),
                    test_history[i].values.flatten()
                )

    def test_passing_of_args(self):
        algo = BatchTransformAlgorithm(1, kwarg='str')
        self.assertEqual(algo.args, (1,))
        self.assertEqual(algo.kwargs, {'kwarg': 'str'})

        algo.run(self.source)
        expected_item = ((1, ), {'kwarg': 'str'})
        self.assertEqual(
            algo.history_return_args,
            [
                # 1990-01-03 - window not full
                None,
                # 1990-01-04 - window not full
                None,
                # 1990-01-05 - window not full, 3rd event
                None,
                # 1990-01-08 - window now full
                expected_item
            ])
<|endoftext|>


<commit_before>from __future__ import (absolute_import, division, print_function)

import gridgeo

from shapely.geometry import MultiPolygon, Polygon

url = 'http://thredds.cencoos.org/thredds/dodsC/CA_DAS.nc'

grid = gridgeo.GridGeo(
    url,
    standard_name='sea_water_temperature'
    )
npoly = 136500

def test_mesh():
    assert grid.mesh == 'unknown_1d'

def test__str__():
    assert grid.__str__() == grid.mesh

def test_outline():
    assert isinstance(grid.outline, (MultiPolygon, Polygon))

def test_polygons():
    assert isinstance(grid.polygons, MultiPolygon)

def test_polygons_len():
    assert len(grid.polygons) == npoly

def test_geo_interface():
    assert isinstance(grid.__geo_interface__, dict)

def test_to_geojson():
    geojson = grid.to_geojson(float_precision=2)
    assert geojson['type'] == 'Feature'
    assert isinstance(geojson, dict)
    assert geojson['geometry']['type'] == 'MultiPolygon'
    assert geojson['properties'] == {
        'description': '',
        'fill': '555555',
        'fill-opacity': 0.6,
        'marker-color': '7e7e7e',
        'marker-size': 'medium',
        'marker-symbol': '',
        'stroke': '555555',
        'stroke-opacity': 1,
        'stroke-width': 2,
        'title': grid.mesh
        }
    coords = geojson['geometry']['coordinates']
    assert len(coords) == npoly
    assert len(coords[0][0]) == 5  # squares are 4+1
    assert str(coords[0][0][0][0]) == '-127.5'
<commit_msg>Fixing style errors.<commit_after>from __future__ import (absolute_import, division, print_function)

import gridgeo

from shapely.geometry import MultiPolygon, Polygon

url = 'http://thredds.cencoos.org/thredds/dodsC/CA_DAS.nc'

grid = gridgeo.GridGeo(
    url,
    standard_name='sea_water_temperature'
)
npoly = 136500

def test_mesh():
    assert grid.mesh == 'unknown_1d'

def test__str__():
    assert grid.__str__() == grid.mesh

def test_outline():
    assert isinstance(grid.outline, (MultiPolygon, Polygon))

def test_polygons():
    assert isinstance(grid.polygons, MultiPolygon)

def test_polygons_len():
    assert len(grid.polygons) == npoly

def test_geo_interface():
    assert isinstance(grid.__geo_interface__, dict)

def test_to_geojson():
    geojson = grid.to_geojson(float_precision=2)
    assert geojson['type'] == 'Feature'
    assert isinstance(geojson, dict)
    assert geojson['geometry']['type'] == 'MultiPolygon'
    assert geojson['properties'] == {
        'description': '',
        'fill': '555555',
        'fill-opacity': 0.6,
        'marker-color': '7e7e7e',
        'marker-size': 'medium',
        'marker-symbol': '',
        'stroke': '555555',
        'stroke-opacity': 1,
        'stroke-width': 2,
        'title': grid.mesh
    }
    coords = geojson['geometry']['coordinates']
    assert len(coords) == npoly
    assert len(coords[0][0]) == 5  # squares are 4+1
    assert str(coords[0][0][0][0]) == '-127.5'
<|endoftext|>


<commit_before># Copyright 2013-2016 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    import unittest2 as unittest
except ImportError:
    import unittest  # noqa

import six

from cassandra.query import BatchStatement, SimpleStatement

class BatchStatementTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a PR

    def test_clear(self):
        keyspace = 'keyspace'
        routing_key = 'routing_key'
        custom_payload = {'key': six.b('value')}

        ss = SimpleStatement('whatever', keyspace=keyspace, routing_key=routing_key, custom_payload=custom_payload)

        batch = BatchStatement()
        batch.add(ss)

        self.assertTrue(batch._statements_and_parameters)
        self.assertEqual(batch.keyspace, keyspace)
        self.assertEqual(batch.routing_key, routing_key)
        self.assertEqual(batch.custom_payload, custom_payload)

        batch.clear()
        self.assertFalse(batch._statements_and_parameters)
        self.assertIsNone(batch.keyspace)
        self.assertIsNone(batch.routing_key)
        self.assertFalse(batch.custom_payload)

        batch.add(ss)

    def test_clear_empty(self):
        batch = BatchStatement()
        batch.clear()
        self.assertFalse(batch._statements_and_parameters)
        self.assertIsNone(batch.keyspace)
        self.assertIsNone(batch.routing_key)
        self.assertFalse(batch.custom_payload)

        batch.add('something')

    def test_add_all(self):
        batch = BatchStatement()
        statements = ['%s'] * 10
        parameters = [(i,) for i in range(10)]
        batch.add_all(statements, parameters)
        bound_statements = [t[1] for t in batch._statements_and_parameters]
        str_parameters = [str(i) for i in range(10)]
        self.assertEqual(bound_statements, str_parameters)
<commit_msg>add smoke test for BatchStatement.__len__<commit_after># Copyright 2013-2016 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

try:
    import unittest2 as unittest
except ImportError:
    import unittest  # noqa

import six

from cassandra.query import BatchStatement, SimpleStatement

class BatchStatementTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a PR

    def test_clear(self):
        keyspace = 'keyspace'
        routing_key = 'routing_key'
        custom_payload = {'key': six.b('value')}

        ss = SimpleStatement('whatever', keyspace=keyspace, routing_key=routing_key, custom_payload=custom_payload)

        batch = BatchStatement()
        batch.add(ss)

        self.assertTrue(batch._statements_and_parameters)
        self.assertEqual(batch.keyspace, keyspace)
        self.assertEqual(batch.routing_key, routing_key)
        self.assertEqual(batch.custom_payload, custom_payload)

        batch.clear()
        self.assertFalse(batch._statements_and_parameters)
        self.assertIsNone(batch.keyspace)
        self.assertIsNone(batch.routing_key)
        self.assertFalse(batch.custom_payload)

        batch.add(ss)

    def test_clear_empty(self):
        batch = BatchStatement()
        batch.clear()
        self.assertFalse(batch._statements_and_parameters)
        self.assertIsNone(batch.keyspace)
        self.assertIsNone(batch.routing_key)
        self.assertFalse(batch.custom_payload)

        batch.add('something')

    def test_add_all(self):
        batch = BatchStatement()
        statements = ['%s'] * 10
        parameters = [(i,) for i in range(10)]
        batch.add_all(statements, parameters)
        bound_statements = [t[1] for t in batch._statements_and_parameters]
        str_parameters = [str(i) for i in range(10)]
        self.assertEqual(bound_statements, str_parameters)

    def test_len(self):
        for n in 0, 10, 100:
            batch = BatchStatement()
            batch.add_all(statements=['%s'] * n,
                          parameters=[(i,) for i in range(n)])
            self.assertEqual(len(batch), n)
<|endoftext|>


<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
unittest class used in Selenium tests. Every test_cluster_* inherits methods
from ClusterTest.

@author: Ioannis Stenos, Nick Vrionis
'''

from selenium import webdriver
import sys
from os.path import join, dirname, abspath
sys.path.append(join(dirname(abspath(__file__)), '../..'))
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from ConfigParser import RawConfigParser, NoSectionError
import unittest, time, re

BASE_DIR = join(dirname(abspath(__file__)), "../..")

class ClusterTest(unittest.TestCase):
    '''
    setUp method is common for all test_cluster tests.
    Defines the path to okeanos token and the base url for
    the selenium test.
    '''
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(30)
        self.verificationErrors = []
        self.accept_next_alert = True
        parser = RawConfigParser()
        config_file = join(BASE_DIR, '.private/.config.txt')
        self.name = 'testcluster'
        parser.read(config_file)
        try:
            self.token = parser.get('cloud \"~okeanos\"', 'token')
            self.auth_url = parser.get('cloud \"~okeanos\"', 'url')
            self.base_url = parser.get('cloud \"~okeanos\"', 'app_url')
        except NoSectionError:
            self.token = 'INVALID_TOKEN'
            self.auth_url = "INVALID_AUTH_URL"
            self.base_url = "INVALID_ESCIENCE_APP_HOST_URL"
            print 'Current authentication details are kept off source control. ' \
                  '\nUpdate your .config.txt file in <projectroot>/.private/'

    def login(self):
        '''Method used for login by all test_cluster tests'''
        driver = self.driver
        driver.get(self.base_url + "#/homepage")
        driver.find_element_by_css_selector("button[type=\"submit\"]").click()
        driver.find_element_by_id("token").clear()
        driver.find_element_by_id("token").send_keys(self.token)
        driver.find_element_by_css_selector("button[type=\"login\"]").click()
        for i in range(60):
            try:
                if self.is_element_present(By.ID, "CreateHadoopCluster"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        driver.find_element_by_id("CreateHadoopCluster").click()
        for i in range(60):
            try:
                if self.is_element_present(By.XPATH, "//div[@id='sidebar']/p/select"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        return driver

    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException, e: return False
        return True

    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException, e: return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

    def calculate_cluster_resources(self, resource_list, available_resource):
        '''
        Method used by test_cluster_cpu, test_cluster_memory and
        test_cluster_disk to calculate the resources needed to be binded
        during each test in cluster/create screen. Those resources are
        the buttons pressed by selenium test and are not really binded
        in ~okeanos. The result is that much less real resources are binded
        in ~okeanos for the tests.
        '''
        avail = available_resource
        cluster_size = 0
        vms = []
        # Create a vms list with values the combinations of resource size
        # and number of vms we will bind in selenium with a particular,
        # resource flavor e.g for test_cluster_disk vms = [{100:2}, {80:1}]
        # means 2 vms with 100 disk size each and 1 vm with 80 disk size.
        for resource in reversed(resource_list):
            if (available_resource/resource) >= 1:
                vms.append({resource: available_resource/resource})
                if available_resource%resource ==0:
                    break
                available_resource = available_resource - resource*(available_resource/resource)
        # If the vms list has two or more elements
        if len(vms) >= 2:
            # Find the remaining resource that we will bind in ~okeanos for the test.
            remaining_resource = avail - vms[0].values()[0] * resource_list[len(resource_list)-1] - vms[1].values()[0] * resource_list[len(resource_list)-2]
            # Calculate the cluster_size we will use as input in selenium
            cluster_size = vms[0].values()[0] + vms[1].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    # Select the buttons selenium will press
                    # in create_cluster screen
                    slave = str(resource_list.index(vms[0].values()[0]) + 1)
                    master = str(resource_list.index(vms[1].values()[0]))
                    break
        # If the vms list has zero elements
        elif len(vms) == 0:
            raise RuntimeError
        # If the vms list has only one element
        else:
            remaining_resource = 0
            cluster_size = cluster_size + vms[0].values()[0]
            for resource in resource_list:
                if resource == vms[0].keys()[0]:
                    slave = str(resource_list.index(vms[0].keys()[0]) + 1)
                    master = str(resource_list.index(vms[0].keys()[0]) + 1)
                    break
        return cluster_size, master, slave, remaining_resource
<commit_msg>ES-149 ClusterTest fix<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

'''
unittest class used in Selenium tests. Every test_cluster_* inherits methods
from ClusterTest.

@author: Ioannis Stenos, Nick Vrionis
'''

from selenium import webdriver
import sys
from os.path import join, dirname, abspath
sys.path.append(join(dirname(abspath(__file__)), '../..'))
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
from selenium.common.exceptions import NoSuchElementException
from selenium.common.exceptions import NoAlertPresentException
from ConfigParser import RawConfigParser, NoSectionError
import unittest, time, re

BASE_DIR = join(dirname(abspath(__file__)), "../..")

class ClusterTest(unittest.TestCase):
    '''
    setUp method is common for all test_cluster tests.
    Defines the path to okeanos token and the base url for
    the selenium test.
    '''
    def setUp(self):
        self.driver = webdriver.Firefox()
        self.driver.implicitly_wait(30)
        self.verificationErrors = []
        self.accept_next_alert = True
        parser = RawConfigParser()
        config_file = join(BASE_DIR, '.private/.config.txt')
        self.name = 'testcluster'
        parser.read(config_file)
        try:
            self.token = parser.get('cloud \"~okeanos\"', 'token')
            self.auth_url = parser.get('cloud \"~okeanos\"', 'url')
            self.base_url = parser.get('cloud \"~okeanos\"', 'app_url')
        except NoSectionError:
            self.token = 'INVALID_TOKEN'
            self.auth_url = "INVALID_AUTH_URL"
            self.base_url = "INVALID_ESCIENCE_APP_HOST_URL"
            print 'Current authentication details are kept off source control. ' \
                  '\nUpdate your .config.txt file in <projectroot>/.private/'

    def login(self):
        '''Method used for login by all test_cluster tests'''
        driver = self.driver
        driver.get(self.base_url + "#/homepage")
        driver.find_element_by_css_selector("button[type=\"submit\"]").click()
        driver.find_element_by_id("token").clear()
        driver.find_element_by_id("token").send_keys(self.token)
        driver.find_element_by_css_selector("button[type=\"login\"]").click()
        for i in range(60):
            try:
                if self.is_element_present(By.ID, "CreateHadoopCluster"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        driver.find_element_by_id("CreateHadoopCluster").click()
        for i in range(60):
            try:
                if self.is_element_present(By.XPATH, "//div[@id='sidebar']/p/select"): break
            except: pass
            time.sleep(1)
        else: self.fail("time out")
        return driver

    def is_element_present(self, how, what):
        try: self.driver.find_element(by=how, value=what)
        except NoSuchElementException, e: return False
        return True

    def is_alert_present(self):
        try: self.driver.switch_to_alert()
        except NoAlertPresentException, e: return False
        return True

    def close_alert_and_get_its_text(self):
        try:
            alert = self.driver.switch_to_alert()
            alert_text = alert.text
            if self.accept_next_alert:
                alert.accept()
            else:
                alert.dismiss()
            return alert_text
        finally: self.accept_next_alert = True

    def tearDown(self):
        self.driver.quit()
        self.assertEqual([], self.verificationErrors)

    def calculate_cluster_resources(self, resource_list, available_resource):
        '''
        Method used by test_cluster_cpu, test_cluster_memory and
        test_cluster_disk to calculate the resources needed to be binded
        during each test in cluster/create screen. Those resources are
        the buttons pressed by selenium test and are not really binded
        in ~okeanos. The result is that much less real resources are binded
        in ~okeanos for the tests.
        '''
        avail = available_resource
        cluster_size = 0
        vms = []
        # Create a vms list with values the combinations of resource size
        # and number of vms we will bind in selenium with a particular,
        # resource flavor e.g for test_cluster_disk vms = [{100:2}, {80:1}]
        # means 2 vms with 100 disk size each and 1 vm with 80 disk size.
        for resource in reversed(resource_list):
            if (available_resource/resource) >= 1:
                vms.append({resource: available_resource/resource})
                if available_resource%resource ==0:
                    break
                available_resource = available_resource - resource*(available_resource/resource)
        # If the vms list has two or more elements
        if len(vms) >= 2:
            # Find the remaining resource that we will bind in ~okeanos for the test.
            remaining_resource = avail - vms[0].values()[0] * resource_list[len(resource_list)-1] - vms[1].values()[0] * resource_list[len(resource_list)-2]
            # Calculate the cluster_size we will use as input in selenium
            cluster_size = vms[0].values()[0] + vms[1].values()[0]
            # Select the buttons selenium will press
            # in create_cluster screen
            slave = str(resource_list.index(vms[0].keys()[0]) + 1)
            master = str(resource_list.index(vms[1].keys()[0]))

        # If the vms list has zero elements
        elif len(vms) == 0:
            raise RuntimeError
        # If the vms list has only one element
        else:
            remaining_resource = 0
            cluster_size = cluster_size + vms[0].values()[0]
            slave = str(resource_list.index(vms[0].keys()[0]) + 1)
            master = str(resource_list.index(vms[0].keys()[0]) + 1)

        return cluster_size, master, slave, remaining_resource
<|endoftext|>


<commit_before>import logging

from ..orchestra import run

log = logging.getLogger(__name__)

def task(ctx, config):
    """
    Sync or skew clock

    This will initially sync the clocks.  Eventually it should let us also
    skew by some number of seconds.

    example:

    tasks:
    - clock:
    - ceph:
    - interactive:

    to sync.

    To skew, we should allow something like:

    tasks:
    - clock:
        mon.0: -2
        client.1: 2
    - ceph:
    - interactive:

    """

    ctx.cluster.run(
        args=[
            'sudo',
            'service', 'ntp', 'stop',
            run.Raw(';'),
            'sudo',
            'ntpdate',
            'clock1.dreamhost.com',
            'clock2.dreamhost.com',
            'clock3.dreamhost.com',
            run.Raw(';'),
            'sudo',
            'service', 'ntp', 'start',
            run.Raw('||'),
            'true'
            ],
        )

    # TODO do the skew
    #for role, skew in config.iteritems():
<commit_msg>clock: print skew with ntp servers to log to help debug time issues<commit_after>import logging
import contextlib

from ..orchestra import run

log = logging.getLogger(__name__)

@contextlib.contextmanager
def task(ctx, config):
    """
    Sync or skew clock

    This will initially sync the clocks.  Eventually it should let us also
    skew by some number of seconds.

    example:

    tasks:
    - clock:
    - ceph:
    - interactive:

    to sync.

    """

    log.info('Syncing clocks and checking initial clock skew...')
    for rem in ctx.cluster.remotes.iterkeys():
        rem.run(
            args=[
                'sudo',
                'service', 'ntp', 'stop',
                run.Raw(';'),
                'sudo',
                'ntpdate',
                'clock1.dreamhost.com',
                'clock2.dreamhost.com',
                'clock3.dreamhost.com',
                run.Raw(';'),
                'sudo',
                'service', 'ntp', 'start',
                run.Raw(';'),
                'ntpdc', '-p',
                ],
            logger=log.getChild(rem.name),
        )

    try:
        yield

    finally:
        log.info('Checking final clock skew...')
        for rem in ctx.cluster.remotes.iterkeys():
            rem.run(
                args=[
                    'ntpdc', '-p',
                    ],
                logger=log.getChild(rem.name),
                )
<|endoftext|>


<commit_before># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django import forms
from django.utils.encoding import force_text
from django.utils.html import format_html

class NamelessTextInput(forms.TextInput):
    """TextInput which does not render a ``name`` attribute."""

    def render(self, name, value, attrs=None):
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, type=self.input_type)
        if value != '':
            # Only add the 'value' attribute if a value is non-empty.
            final_attrs['value'] = force_text(self._format_value(value))
        return format_html('<input{0} />', forms.utils.flatatt(final_attrs))
<commit_msg>modified import in widgets for django 1.6 (util was renamed to utils in 1.7)<commit_after># -*- coding: utf-8 -*-
from __future__ import absolute_import, unicode_literals
from django import forms
try:
    from django.forms.utils import flatatt
except ImportError:
    from django.forms.util import flatatt
from django.utils.encoding import force_text
from django.utils.html import format_html

class NamelessTextInput(forms.TextInput):
    """TextInput which does not render a ``name`` attribute."""

    def render(self, name, value, attrs=None):
        if value is None:
            value = ''
        final_attrs = self.build_attrs(attrs, type=self.input_type)
        if value != '':
            # Only add the 'value' attribute if a value is non-empty.
            final_attrs['value'] = force_text(self._format_value(value))
        return format_html('<input{0} />', flatatt(final_attrs))
<|endoftext|>