<commit_before>from __future__ import unicode_literals
from __future__ import absolute_import
import os

from .. import unittest
import mock

import docker

from fig import Service
from fig.service import (
    ConfigError,
    split_port,
    parse_volume_spec,
    build_volume_binding,
)

class ServiceTest(unittest.TestCase):

    def setUp(self):
        self.mock_client = mock.create_autospec(docker.Client)

    def test_name_validations(self):
        self.assertRaises(ConfigError, lambda: Service(name=''))

        self.assertRaises(ConfigError, lambda: Service(name=' '))
        self.assertRaises(ConfigError, lambda: Service(name='/'))
        self.assertRaises(ConfigError, lambda: Service(name='!'))
        self.assertRaises(ConfigError, lambda: Service(name='\xe2'))
        self.assertRaises(ConfigError, lambda: Service(name='_'))
        self.assertRaises(ConfigError, lambda: Service(name='____'))
        self.assertRaises(ConfigError, lambda: Service(name='foo_bar'))
        self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__'))

        Service('a')
        Service('foo')

    def test_project_validation(self):
        self.assertRaises(ConfigError, lambda: Service(name='foo', project='_'))
        Service(name='foo', project='bar')

    def test_config_validation(self):
        self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000']))
        Service(name='foo', ports=['8000'])

    def test_split_port_with_host_ip(self):
        internal_port, external_port = split_port("127.0.0.1:1000:2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, ("127.0.0.1", "1000"))

    def test_split_port_with_protocol(self):
        internal_port, external_port = split_port("127.0.0.1:1000:2000/udp")
        self.assertEqual(internal_port, "2000/udp")
        self.assertEqual(external_port, ("127.0.0.1", "1000"))

    def test_split_port_with_host_ip_no_port(self):
        internal_port, external_port = split_port("127.0.0.1::2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, ("127.0.0.1", None))

    def test_split_port_with_host_port(self):
        internal_port, external_port = split_port("1000:2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, "1000")

    def test_split_port_no_host_port(self):
        internal_port, external_port = split_port("2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, None)

    def test_split_port_invalid(self):
        with self.assertRaises(ConfigError):
            split_port("0.0.0.0:1000:2000:tcp")

    def test_split_domainname_none(self):
        service = Service('foo', hostname='name', client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertFalse('domainname' in opts, 'domainname')

    def test_split_domainname_fqdn(self):
        service = Service('foo',
                hostname='name.domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_split_domainname_both(self):
        service = Service('foo',
                hostname='name',
                domainname='domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_split_domainname_weird(self):
        service = Service('foo',
                hostname='name.sub',
                domainname='domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name.sub', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_get_container_not_found(self):
        mock_client = mock.create_autospec(docker.Client)
        mock_client.containers.return_value = []
        service = Service('foo', client=mock_client)

        self.assertRaises(ValueError, service.get_container)

    @mock.patch('fig.service.Container', autospec=True)
    def test_get_container(self, mock_container_class):
        mock_client = mock.create_autospec(docker.Client)
        container_dict = dict(Name='default_foo_2')
        mock_client.containers.return_value = [container_dict]
        service = Service('foo', client=mock_client)

        container = service.get_container(number=2)
        self.assertEqual(container, mock_container_class.from_ps.return_value)
        mock_container_class.from_ps.assert_called_once_with(
            mock_client, container_dict)

class ServiceVolumesTest(unittest.TestCase):

    def test_parse_volume_spec_only_one_path(self):
        spec = parse_volume_spec('/the/volume')
        self.assertEqual(spec, (None, '/the/volume', 'rw'))

    def test_parse_volume_spec_internal_and_external(self):
        spec = parse_volume_spec('external:interval')
        self.assertEqual(spec, ('external', 'interval', 'rw'))

    def test_parse_volume_spec_with_mode(self):
        spec = parse_volume_spec('external:interval:ro')
        self.assertEqual(spec, ('external', 'interval', 'ro'))

    def test_parse_volume_spec_too_many_parts(self):
        with self.assertRaises(ConfigError):
            parse_volume_spec('one:two:three:four')

    def test_parse_volume_bad_mode(self):
        with self.assertRaises(ConfigError):
            parse_volume_spec('one:two:notrw')

    def test_build_volume_binding(self):
        binding = build_volume_binding(parse_volume_spec('/outside:/inside'))
        self.assertEqual(
            binding,
            ('/outside', dict(bind='/inside', ro=False)))

    @mock.patch.dict(os.environ)
    def test_build_volume_binding_with_environ(self):
        os.environ['VOLUME_PATH'] = '/opt'
        binding = build_volume_binding(parse_volume_spec('${VOLUME_PATH}:/opt'))
        self.assertEqual(binding, ('/opt', dict(bind='/opt', ro=False)))

    @mock.patch.dict(os.environ)
    def test_building_volume_binding_with_home(self):
        os.environ['HOME'] = '/home/user'
        binding = build_volume_binding(parse_volume_spec('~:/home/user'))
        self.assertEqual(
            binding,
            ('/home/user', dict(bind='/home/user', ro=False)))
<commit_msg>Add unit test for Service.pull<commit_after>from __future__ import unicode_literals
from __future__ import absolute_import
import os

from .. import unittest
import mock

import docker

from fig import Service
from fig.service import (
    ConfigError,
    split_port,
    parse_volume_spec,
    build_volume_binding,
)

class ServiceTest(unittest.TestCase):

    def setUp(self):
        self.mock_client = mock.create_autospec(docker.Client)

    def test_name_validations(self):
        self.assertRaises(ConfigError, lambda: Service(name=''))

        self.assertRaises(ConfigError, lambda: Service(name=' '))
        self.assertRaises(ConfigError, lambda: Service(name='/'))
        self.assertRaises(ConfigError, lambda: Service(name='!'))
        self.assertRaises(ConfigError, lambda: Service(name='\xe2'))
        self.assertRaises(ConfigError, lambda: Service(name='_'))
        self.assertRaises(ConfigError, lambda: Service(name='____'))
        self.assertRaises(ConfigError, lambda: Service(name='foo_bar'))
        self.assertRaises(ConfigError, lambda: Service(name='__foo_bar__'))

        Service('a')
        Service('foo')

    def test_project_validation(self):
        self.assertRaises(ConfigError, lambda: Service(name='foo', project='_'))
        Service(name='foo', project='bar')

    def test_config_validation(self):
        self.assertRaises(ConfigError, lambda: Service(name='foo', port=['8000']))
        Service(name='foo', ports=['8000'])

    def test_split_port_with_host_ip(self):
        internal_port, external_port = split_port("127.0.0.1:1000:2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, ("127.0.0.1", "1000"))

    def test_split_port_with_protocol(self):
        internal_port, external_port = split_port("127.0.0.1:1000:2000/udp")
        self.assertEqual(internal_port, "2000/udp")
        self.assertEqual(external_port, ("127.0.0.1", "1000"))

    def test_split_port_with_host_ip_no_port(self):
        internal_port, external_port = split_port("127.0.0.1::2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, ("127.0.0.1", None))

    def test_split_port_with_host_port(self):
        internal_port, external_port = split_port("1000:2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, "1000")

    def test_split_port_no_host_port(self):
        internal_port, external_port = split_port("2000")
        self.assertEqual(internal_port, "2000")
        self.assertEqual(external_port, None)

    def test_split_port_invalid(self):
        with self.assertRaises(ConfigError):
            split_port("0.0.0.0:1000:2000:tcp")

    def test_split_domainname_none(self):
        service = Service('foo', hostname='name', client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertFalse('domainname' in opts, 'domainname')

    def test_split_domainname_fqdn(self):
        service = Service('foo',
                hostname='name.domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_split_domainname_both(self):
        service = Service('foo',
                hostname='name',
                domainname='domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_split_domainname_weird(self):
        service = Service('foo',
                hostname='name.sub',
                domainname='domain.tld',
                client=self.mock_client)
        self.mock_client.containers.return_value = []
        opts = service._get_container_create_options({})
        self.assertEqual(opts['hostname'], 'name.sub', 'hostname')
        self.assertEqual(opts['domainname'], 'domain.tld', 'domainname')

    def test_get_container_not_found(self):
        mock_client = mock.create_autospec(docker.Client)
        mock_client.containers.return_value = []
        service = Service('foo', client=mock_client)

        self.assertRaises(ValueError, service.get_container)

    @mock.patch('fig.service.Container', autospec=True)
    def test_get_container(self, mock_container_class):
        mock_client = mock.create_autospec(docker.Client)
        container_dict = dict(Name='default_foo_2')
        mock_client.containers.return_value = [container_dict]
        service = Service('foo', client=mock_client)

        container = service.get_container(number=2)
        self.assertEqual(container, mock_container_class.from_ps.return_value)
        mock_container_class.from_ps.assert_called_once_with(
            mock_client, container_dict)

    @mock.patch('fig.service.log', autospec=True)
    def test_pull_image(self, mock_log):
        service = Service('foo', client=self.mock_client, image='someimage:sometag')
        service.pull(insecure_registry=True)
        self.mock_client.pull.assert_called_once_with('someimage:sometag', insecure_registry=True)
        mock_log.info.assert_called_once_with('Pulling foo (someimage:sometag)...')

class ServiceVolumesTest(unittest.TestCase):

    def test_parse_volume_spec_only_one_path(self):
        spec = parse_volume_spec('/the/volume')
        self.assertEqual(spec, (None, '/the/volume', 'rw'))

    def test_parse_volume_spec_internal_and_external(self):
        spec = parse_volume_spec('external:interval')
        self.assertEqual(spec, ('external', 'interval', 'rw'))

    def test_parse_volume_spec_with_mode(self):
        spec = parse_volume_spec('external:interval:ro')
        self.assertEqual(spec, ('external', 'interval', 'ro'))

    def test_parse_volume_spec_too_many_parts(self):
        with self.assertRaises(ConfigError):
            parse_volume_spec('one:two:three:four')

    def test_parse_volume_bad_mode(self):
        with self.assertRaises(ConfigError):
            parse_volume_spec('one:two:notrw')

    def test_build_volume_binding(self):
        binding = build_volume_binding(parse_volume_spec('/outside:/inside'))
        self.assertEqual(
            binding,
            ('/outside', dict(bind='/inside', ro=False)))

    @mock.patch.dict(os.environ)
    def test_build_volume_binding_with_environ(self):
        os.environ['VOLUME_PATH'] = '/opt'
        binding = build_volume_binding(parse_volume_spec('${VOLUME_PATH}:/opt'))
        self.assertEqual(binding, ('/opt', dict(bind='/opt', ro=False)))

    @mock.patch.dict(os.environ)
    def test_building_volume_binding_with_home(self):
        os.environ['HOME'] = '/home/user'
        binding = build_volume_binding(parse_volume_spec('~:/home/user'))
        self.assertEqual(
            binding,
            ('/home/user', dict(bind='/home/user', ro=False)))
<|endoftext|>


<commit_before>'''
The retro analysis module
Runs analysis after a lab run using existing data files
e.g. yarn retro_analyze data/reinforce_cartpole_2018_01_22_211751
'''
from slm_lab.experiment import analysis
from slm_lab.lib import logger, util
from slm_lab.spec import spec_util
import numpy as np
import os
import pydash as ps
import regex as re

logger = logger.get_logger(__name__)

def session_data_from_file(predir, trial_index, session_index, ckpt=None, prefix=''):
    '''Build session.session_data from file'''
    ckpt_str = '' if ckpt is None else f'_ckpt-{ckpt}'
    for filename in os.listdir(predir):
        if filename.endswith(f'_t{trial_index}_s{session_index}{ckpt_str}_{prefix}session_df.csv'):
            filepath = f'{predir}/{filename}'
            session_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0)
            session_data = util.session_df_to_data(session_df)
            return session_data

def session_datas_from_file(predir, trial_spec, trial_index, ckpt=None):
    '''Return a dict of {session_index: session_data} for a trial'''
    session_datas = {}
    for s in range(trial_spec['meta']['max_session']):
        session_data = session_data_from_file(predir, trial_index, s, ckpt)
        if session_data is not None:
            session_datas[s] = session_data
    return session_datas

def session_data_dict_from_file(predir, trial_index, ckpt=None):
    '''Build trial.session_data_dict from file'''
    ckpt_str = '' if ckpt is None else f'_ckpt-{ckpt}'
    session_data_dict = {}
    for filename in os.listdir(predir):
        if f'_t{trial_index}_' in filename and filename.endswith(f'{ckpt_str}_session_fitness_df.csv'):
            filepath = f'{predir}/{filename}'
            fitness_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0, dtype=np.float32)
            util.fix_multi_index_dtype(fitness_df)
            session_index = fitness_df.index[0]
            session_data_dict[session_index] = fitness_df
    return session_data_dict

def session_data_dict_for_dist(spec, info_space):
    '''Method to retrieve session_datas (fitness df, so the same as session_data_dict above) when a trial with distributed sessions is done, to avoid messy multiprocessing data communication'''
    prepath = util.get_prepath(spec, info_space)
    predir, _, _, _, _, _ = util.prepath_split(prepath)
    session_datas = session_data_dict_from_file(predir, info_space.get('trial'), ps.get(info_space, 'ckpt'))
    session_datas = [session_datas[k] for k in sorted(session_datas.keys())]
    return session_datas

def trial_data_dict_from_file(predir):
    '''Build experiment.trial_data_dict from file'''
    trial_data_dict = {}
    for filename in os.listdir(predir):
        if filename.endswith('_trial_data.json'):
            filepath = f'{predir}/{filename}'
            exp_trial_data = util.read(filepath)
            trial_index = exp_trial_data.pop('trial_index')
            trial_data_dict[trial_index] = exp_trial_data
    return trial_data_dict

'''
Interface retro methods
'''

def analyze_eval_trial(spec, info_space, predir):
    '''Create a trial and run analysis to get the trial graph and other trial data'''
    from slm_lab.experiment.control import Trial
    trial = Trial(spec, info_space)
    trial.session_data_dict = session_data_dict_from_file(predir, trial.index, ps.get(info_space, 'ckpt'))
    analysis.analyze_trial(trial)

def run_parallel_eval(spec, info_space, ckpt):
    '''
    Calls a subprocess to run lab in eval mode with the constructed ckpt prepath, same as how one would manually run the bash cmd
    @example

    python run_lab.py data/dqn_cartpole_2018_12_19_224811/dqn_cartpole_t0_spec.json dqn_cartpole eval@dqn_cartpole_t0_s1_ckpt-epi10-totalt1000
    '''
    prepath_t = util.get_prepath(spec, info_space, unit='trial')
    prepath_s = util.get_prepath(spec, info_space, unit='session')
    predir, _, prename, spec_name, _, _ = util.prepath_split(prepath_s)
    cmd = f'python run_lab.py {prepath_t}_spec.json {spec_name} eval@{prename}_ckpt-{ckpt}'
    logger.info(f'Running parallel eval for ckpt-{ckpt}')
    return util.run_cmd(cmd)

def run_parallel_eval(session, agent, env):
    '''Plugin to session to run parallel eval for train mode'''
    if util.get_lab_mode() == 'train':
        ckpt = f'epi{env.clock.epi}-totalt{env.clock.total_t}'
        agent.save(ckpt=ckpt)
        # set reference to eval process for handling
        session.eval_proc = run_parallel_eval(session.spec, session.info_space, ckpt)

def try_wait_parallel_eval(session):
    '''Plugin to wait for session's final parallel eval if any'''
    if hasattr(session, 'eval_proc') and session.eval_proc is not None:  # wait for final eval before closing
        util.run_cmd_wait(session.eval_proc)
        session_retro_eval(session)  # rerun failed eval

def run_parallel_eval_from_prepath(prepath):
    '''Used by retro_eval'''
    spec, info_space = util.prepath_to_spec_info_space(prepath)
    ckpt = util.find_ckpt(prepath)
    return run_parallel_eval(spec, info_space, ckpt)

def run_wait_eval(prepath):
    '''Used by retro_eval'''
    eval_proc = run_parallel_eval_from_prepath(prepath)
    util.run_cmd_wait(eval_proc)

def retro_analyze_sessions(predir):
    '''Retro-analyze all session level datas.'''
    logger.info('Retro-analyzing sessions from file')
    from slm_lab.experiment.control import Session, SpaceSession
    for filename in os.listdir(predir):
        # to account for both types of session_df
        if filename.endswith('_session_df.csv'):
            body_df_kind = 'eval'  # from body.eval_df
            prefix = ''
            is_session_df = True
        elif filename.endswith('_trainsession_df.csv'):
            body_df_kind = 'train'  # from body.train_df
            prefix = 'train'
            is_session_df = True
        else:
            is_session_df = False

        if is_session_df:
            prepath = f'{predir}/{filename}'.replace(f'_{prefix}session_df.csv', '')
            spec, info_space = util.prepath_to_spec_info_space(prepath)
            trial_index, session_index = util.prepath_to_idxs(prepath)
            SessionClass = Session if spec_util.is_singleton(spec) else SpaceSession
            session = SessionClass(spec, info_space)
            session_data = session_data_from_file(predir, trial_index, session_index, ps.get(info_space, 'ckpt'), prefix)
            analysis._analyze_session(session, session_data, body_df_kind)

def retro_analyze_trials(predir):
    '''Retro-analyze all trial level datas.'''
    logger.info('Retro-analyzing trials from file')
    from slm_lab.experiment.control import Trial
    for filename in os.listdir(predir):
        if filename.endswith('_trial_df.csv'):
            filepath = f'{predir}/{filename}'
            prepath = filepath.replace('_trial_df.csv', '')
            spec, info_space = util.prepath_to_spec_info_space(prepath)
            trial_index, _ = util.prepath_to_idxs(prepath)
            trial = Trial(spec, info_space)
            trial.session_data_dict = session_data_dict_from_file(predir, trial_index, ps.get(info_space, 'ckpt'))
            trial_fitness_df = analysis.analyze_trial(trial)

            # write trial_data that was written from ray search
            trial_data_filepath = filepath.replace('_trial_df.csv', '_trial_data.json')
            if os.path.exists(trial_data_filepath):
                fitness_vec = trial_fitness_df.iloc[0].to_dict()
                fitness = analysis.calc_fitness(trial_fitness_df)
                trial_data = util.read(trial_data_filepath)
                trial_data.update({
                    **fitness_vec, 'fitness': fitness, 'trial_index': trial_index,
                })
                util.write(trial_data, trial_data_filepath)

def retro_analyze_experiment(predir):
    '''Retro-analyze all experiment level datas.'''
    logger.info('Retro-analyzing experiment from file')
    from slm_lab.experiment.control import Experiment
    _, _, _, spec_name, _, _ = util.prepath_split(predir)
    prepath = f'{predir}/{spec_name}'
    spec, info_space = util.prepath_to_spec_info_space(prepath)
    if 'search' not in spec:
        return
    experiment = Experiment(spec, info_space)
    experiment.trial_data_dict = trial_data_dict_from_file(predir)
    if not ps.is_empty(experiment.trial_data_dict):
        return analysis.analyze_experiment(experiment)

def retro_analyze(predir):
    '''
    Method to analyze experiment from file after it ran.
    Read from files, constructs lab units, run retro analyses on all lab units.
    This method has no side-effects, i.e. doesn't overwrite data it should not.
    @example

    yarn retro_analyze data/reinforce_cartpole_2018_01_22_211751
    '''
    os.environ['PREPATH'] = f'{predir}/retro_analyze'  # to prevent overwriting log file
    logger.info(f'Retro-analyzing {predir}')
    retro_analyze_sessions(predir)
    retro_analyze_trials(predir)
    retro_analyze_experiment(predir)

def retro_eval(predir, session_index=None):
    '''
    Method to run eval sessions by scanning a predir for ckpt files. Used to rerun failed eval sessions.
    @example

    yarn retro_eval data/reinforce_cartpole_2018_01_22_211751
    '''
    logger.info(f'Retro-evaluate sessions from predir {predir}')
    # collect all unique prepaths first
    prepaths = []
    s_filter = '' if session_index is None else f'_s{session_index}_'
    for filename in os.listdir(predir):
        if filename.endswith('model.pth') and s_filter in filename:
            res = re.search('.+epi(\d+)-totalt(\d+)', filename)
            if res is not None:
                prepath = f'{predir}/{res[0]}'
                if prepath not in prepaths:
                    prepaths.append(prepath)
    if ps.is_empty(prepaths):
        return

    logger.info(f'Starting retro eval')
    np.random.shuffle(prepaths)  # so that CUDA_ID by trial/session index is spread out
    rand_spec = util.prepath_to_spec(prepaths[0])  # get any prepath, read its max session
    max_session = rand_spec['meta']['max_session']
    util.parallelize_fn(run_wait_eval, prepaths, num_cpus=max_session)

def session_retro_eval(session):
    '''retro_eval but for session at the end to rerun failed evals'''
    prepath = util.get_prepath(session.spec, session.info_space, unit='session')
    predir, _, _, _, _, _ = util.prepath_split(prepath)
    retro_eval(predir, session.index)
<commit_msg>rename parallel_eval method name conflict<commit_after>'''
The retro analysis module
Runs analysis after a lab run using existing data files
e.g. yarn retro_analyze data/reinforce_cartpole_2018_01_22_211751
'''
from slm_lab.experiment import analysis
from slm_lab.lib import logger, util
from slm_lab.spec import spec_util
import numpy as np
import os
import pydash as ps
import regex as re

logger = logger.get_logger(__name__)

def session_data_from_file(predir, trial_index, session_index, ckpt=None, prefix=''):
    '''Build session.session_data from file'''
    ckpt_str = '' if ckpt is None else f'_ckpt-{ckpt}'
    for filename in os.listdir(predir):
        if filename.endswith(f'_t{trial_index}_s{session_index}{ckpt_str}_{prefix}session_df.csv'):
            filepath = f'{predir}/{filename}'
            session_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0)
            session_data = util.session_df_to_data(session_df)
            return session_data

def session_datas_from_file(predir, trial_spec, trial_index, ckpt=None):
    '''Return a dict of {session_index: session_data} for a trial'''
    session_datas = {}
    for s in range(trial_spec['meta']['max_session']):
        session_data = session_data_from_file(predir, trial_index, s, ckpt)
        if session_data is not None:
            session_datas[s] = session_data
    return session_datas

def session_data_dict_from_file(predir, trial_index, ckpt=None):
    '''Build trial.session_data_dict from file'''
    ckpt_str = '' if ckpt is None else f'_ckpt-{ckpt}'
    session_data_dict = {}
    for filename in os.listdir(predir):
        if f'_t{trial_index}_' in filename and filename.endswith(f'{ckpt_str}_session_fitness_df.csv'):
            filepath = f'{predir}/{filename}'
            fitness_df = util.read(filepath, header=[0, 1, 2, 3], index_col=0, dtype=np.float32)
            util.fix_multi_index_dtype(fitness_df)
            session_index = fitness_df.index[0]
            session_data_dict[session_index] = fitness_df
    return session_data_dict

def session_data_dict_for_dist(spec, info_space):
    '''Method to retrieve session_datas (fitness df, so the same as session_data_dict above) when a trial with distributed sessions is done, to avoid messy multiprocessing data communication'''
    prepath = util.get_prepath(spec, info_space)
    predir, _, _, _, _, _ = util.prepath_split(prepath)
    session_datas = session_data_dict_from_file(predir, info_space.get('trial'), ps.get(info_space, 'ckpt'))
    session_datas = [session_datas[k] for k in sorted(session_datas.keys())]
    return session_datas

def trial_data_dict_from_file(predir):
    '''Build experiment.trial_data_dict from file'''
    trial_data_dict = {}
    for filename in os.listdir(predir):
        if filename.endswith('_trial_data.json'):
            filepath = f'{predir}/{filename}'
            exp_trial_data = util.read(filepath)
            trial_index = exp_trial_data.pop('trial_index')
            trial_data_dict[trial_index] = exp_trial_data
    return trial_data_dict

'''
Interface retro methods
'''

def analyze_eval_trial(spec, info_space, predir):
    '''Create a trial and run analysis to get the trial graph and other trial data'''
    from slm_lab.experiment.control import Trial
    trial = Trial(spec, info_space)
    trial.session_data_dict = session_data_dict_from_file(predir, trial.index, ps.get(info_space, 'ckpt'))
    analysis.analyze_trial(trial)

def parallel_eval(spec, info_space, ckpt):
    '''
    Calls a subprocess to run lab in eval mode with the constructed ckpt prepath, same as how one would manually run the bash cmd
    @example

    python run_lab.py data/dqn_cartpole_2018_12_19_224811/dqn_cartpole_t0_spec.json dqn_cartpole eval@dqn_cartpole_t0_s1_ckpt-epi10-totalt1000
    '''
    prepath_t = util.get_prepath(spec, info_space, unit='trial')
    prepath_s = util.get_prepath(spec, info_space, unit='session')
    predir, _, prename, spec_name, _, _ = util.prepath_split(prepath_s)
    cmd = f'python run_lab.py {prepath_t}_spec.json {spec_name} eval@{prename}_ckpt-{ckpt}'
    logger.info(f'Running parallel eval for ckpt-{ckpt}')
    return util.run_cmd(cmd)

def run_parallel_eval(session, agent, env):
    '''Plugin to session to run parallel eval for train mode'''
    if util.get_lab_mode() == 'train':
        ckpt = f'epi{env.clock.epi}-totalt{env.clock.total_t}'
        agent.save(ckpt=ckpt)
        # set reference to eval process for handling
        session.eval_proc = parallel_eval(session.spec, session.info_space, ckpt)

def try_wait_parallel_eval(session):
    '''Plugin to wait for session's final parallel eval if any'''
    if hasattr(session, 'eval_proc') and session.eval_proc is not None:  # wait for final eval before closing
        util.run_cmd_wait(session.eval_proc)
        session_retro_eval(session)  # rerun failed eval

def run_parallel_eval_from_prepath(prepath):
    '''Used by retro_eval'''
    spec, info_space = util.prepath_to_spec_info_space(prepath)
    ckpt = util.find_ckpt(prepath)
    return parallel_eval(spec, info_space, ckpt)

def run_wait_eval(prepath):
    '''Used by retro_eval'''
    eval_proc = run_parallel_eval_from_prepath(prepath)
    util.run_cmd_wait(eval_proc)

def retro_analyze_sessions(predir):
    '''Retro-analyze all session level datas.'''
    logger.info('Retro-analyzing sessions from file')
    from slm_lab.experiment.control import Session, SpaceSession
    for filename in os.listdir(predir):
        # to account for both types of session_df
        if filename.endswith('_session_df.csv'):
            body_df_kind = 'eval'  # from body.eval_df
            prefix = ''
            is_session_df = True
        elif filename.endswith('_trainsession_df.csv'):
            body_df_kind = 'train'  # from body.train_df
            prefix = 'train'
            is_session_df = True
        else:
            is_session_df = False

        if is_session_df:
            prepath = f'{predir}/{filename}'.replace(f'_{prefix}session_df.csv', '')
            spec, info_space = util.prepath_to_spec_info_space(prepath)
            trial_index, session_index = util.prepath_to_idxs(prepath)
            SessionClass = Session if spec_util.is_singleton(spec) else SpaceSession
            session = SessionClass(spec, info_space)
            session_data = session_data_from_file(predir, trial_index, session_index, ps.get(info_space, 'ckpt'), prefix)
            analysis._analyze_session(session, session_data, body_df_kind)

def retro_analyze_trials(predir):
    '''Retro-analyze all trial level datas.'''
    logger.info('Retro-analyzing trials from file')
    from slm_lab.experiment.control import Trial
    for filename in os.listdir(predir):
        if filename.endswith('_trial_df.csv'):
            filepath = f'{predir}/{filename}'
            prepath = filepath.replace('_trial_df.csv', '')
            spec, info_space = util.prepath_to_spec_info_space(prepath)
            trial_index, _ = util.prepath_to_idxs(prepath)
            trial = Trial(spec, info_space)
            trial.session_data_dict = session_data_dict_from_file(predir, trial_index, ps.get(info_space, 'ckpt'))
            trial_fitness_df = analysis.analyze_trial(trial)

            # write trial_data that was written from ray search
            trial_data_filepath = filepath.replace('_trial_df.csv', '_trial_data.json')
            if os.path.exists(trial_data_filepath):
                fitness_vec = trial_fitness_df.iloc[0].to_dict()
                fitness = analysis.calc_fitness(trial_fitness_df)
                trial_data = util.read(trial_data_filepath)
                trial_data.update({
                    **fitness_vec, 'fitness': fitness, 'trial_index': trial_index,
                })
                util.write(trial_data, trial_data_filepath)

def retro_analyze_experiment(predir):
    '''Retro-analyze all experiment level datas.'''
    logger.info('Retro-analyzing experiment from file')
    from slm_lab.experiment.control import Experiment
    _, _, _, spec_name, _, _ = util.prepath_split(predir)
    prepath = f'{predir}/{spec_name}'
    spec, info_space = util.prepath_to_spec_info_space(prepath)
    if 'search' not in spec:
        return
    experiment = Experiment(spec, info_space)
    experiment.trial_data_dict = trial_data_dict_from_file(predir)
    if not ps.is_empty(experiment.trial_data_dict):
        return analysis.analyze_experiment(experiment)

def retro_analyze(predir):
    '''
    Method to analyze experiment from file after it ran.
    Read from files, constructs lab units, run retro analyses on all lab units.
    This method has no side-effects, i.e. doesn't overwrite data it should not.
    @example

    yarn retro_analyze data/reinforce_cartpole_2018_01_22_211751
    '''
    os.environ['PREPATH'] = f'{predir}/retro_analyze'  # to prevent overwriting log file
    logger.info(f'Retro-analyzing {predir}')
    retro_analyze_sessions(predir)
    retro_analyze_trials(predir)
    retro_analyze_experiment(predir)

def retro_eval(predir, session_index=None):
    '''
    Method to run eval sessions by scanning a predir for ckpt files. Used to rerun failed eval sessions.
    @example

    yarn retro_eval data/reinforce_cartpole_2018_01_22_211751
    '''
    logger.info(f'Retro-evaluate sessions from predir {predir}')
    # collect all unique prepaths first
    prepaths = []
    s_filter = '' if session_index is None else f'_s{session_index}_'
    for filename in os.listdir(predir):
        if filename.endswith('model.pth') and s_filter in filename:
            res = re.search('.+epi(\d+)-totalt(\d+)', filename)
            if res is not None:
                prepath = f'{predir}/{res[0]}'
                if prepath not in prepaths:
                    prepaths.append(prepath)
    if ps.is_empty(prepaths):
        return

    logger.info(f'Starting retro eval')
    np.random.shuffle(prepaths)  # so that CUDA_ID by trial/session index is spread out
    rand_spec = util.prepath_to_spec(prepaths[0])  # get any prepath, read its max session
    max_session = rand_spec['meta']['max_session']
    util.parallelize_fn(run_wait_eval, prepaths, num_cpus=max_session)

def session_retro_eval(session):
    '''retro_eval but for session at the end to rerun failed evals'''
    prepath = util.get_prepath(session.spec, session.info_space, unit='session')
    predir, _, _, _, _, _ = util.prepath_split(prepath)
    retro_eval(predir, session.index)
<|endoftext|>


<commit_before># Copyright 2013-2016 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
try:
    import unittest2 as unittest
except ImportError:
    import unittest  # noqa

from copy import copy
from mock import patch, Mock

from cassandra import ConsistencyLevel, DriverException, Timeout, Unavailable, RequestExecutionException, ReadTimeout, WriteTimeout, CoordinationFailure, ReadFailure, WriteFailure, FunctionFailure, AlreadyExists,\
    InvalidRequest, Unauthorized, AuthenticationFailed, OperationTimedOut, UnsupportedOperation, RequestValidationException, ConfigurationException
from cassandra.cluster import _Scheduler, Session, Cluster, _NOT_SET, default_lbp_factory, \
    ExecutionProfile, _ConfigMode, EXEC_PROFILE_DEFAULT
from cassandra.policies import HostDistance, RetryPolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy
from cassandra.query import SimpleStatement, named_tuple_factory, tuple_factory

class ExceptionTypeTest(unittest.TestCase):

    def test_exception_types(self):
        """
        PYTHON-443
        Sanity check to ensure we don't unintentionally change class hierarchy of exception types
        """
        self.assertTrue(issubclass(Unavailable, DriverException))
        self.assertTrue(issubclass(Unavailable, RequestExecutionException))

        self.assertTrue(issubclass(ReadTimeout, DriverException))
        self.assertTrue(issubclass(ReadTimeout, RequestExecutionException))
        self.assertTrue(issubclass(ReadTimeout, Timeout))

        self.assertTrue(issubclass(WriteTimeout, DriverException))
        self.assertTrue(issubclass(WriteTimeout, RequestExecutionException))
        self.assertTrue(issubclass(WriteTimeout, Timeout))

        self.assertTrue(issubclass(CoordinationFailure, DriverException))
        self.assertTrue(issubclass(CoordinationFailure, RequestExecutionException))

        self.assertTrue(issubclass(ReadFailure, DriverException))
        self.assertTrue(issubclass(ReadFailure, RequestExecutionException))
        self.assertTrue(issubclass(ReadFailure, CoordinationFailure))

        self.assertTrue(issubclass(WriteFailure, DriverException))
        self.assertTrue(issubclass(WriteFailure, RequestExecutionException))
        self.assertTrue(issubclass(WriteFailure, CoordinationFailure))

        self.assertTrue(issubclass(FunctionFailure, DriverException))
        self.assertTrue(issubclass(FunctionFailure, RequestExecutionException))

        self.assertTrue(issubclass(RequestValidationException, DriverException))

        self.assertTrue(issubclass(ConfigurationException, DriverException))
        self.assertTrue(issubclass(ConfigurationException, RequestValidationException))

        self.assertTrue(issubclass(AlreadyExists, DriverException))
        self.assertTrue(issubclass(AlreadyExists, RequestValidationException))
        self.assertTrue(issubclass(AlreadyExists, ConfigurationException))

        self.assertTrue(issubclass(InvalidRequest, DriverException))
        self.assertTrue(issubclass(InvalidRequest, RequestValidationException))

        self.assertTrue(issubclass(Unauthorized, DriverException))
        self.assertTrue(issubclass(Unauthorized, RequestValidationException))

        self.assertTrue(issubclass(AuthenticationFailed, DriverException))

        self.assertTrue(issubclass(OperationTimedOut, DriverException))

        self.assertTrue(issubclass(UnsupportedOperation, DriverException))

class ClusterTest(unittest.TestCase):

    def test_invalid_contact_point_types(self):
        with self.assertRaises(ValueError):
            Cluster(contact_points=[None], protocol_version=4, connect_timeout=1)
        with self.assertRaises(TypeError):
            Cluster(contact_points="not a sequence", protocol_version=4, connect_timeout=1)

    def test_requests_in_flight_threshold(self):
        d = HostDistance.LOCAL
        mn = 3
        mx = 5
        c = Cluster(protocol_version=2)
        c.set_min_requests_per_connection(d, mn)
        c.set_max_requests_per_connection(d, mx)
        # min underflow, max, overflow
        for n in (-1, mx, 127):
            self.assertRaises(ValueError, c.set_min_requests_per_connection, d, n)
        # max underflow, under min, overflow
        for n in (0, mn, 128):
            self.assertRaises(ValueError, c.set_max_requests_per_connection, d, n)

class SchedulerTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a ticket

    @patch('time.time', return_value=3)  # always queue at same time
    @patch('cassandra.cluster._Scheduler.run')  # don't actually run the thread
    def test_event_delay_timing(self, *_):
        """
        Schedule something with a time collision to make sure the heap comparison works

        PYTHON-473
        """
        sched = _Scheduler(None)
        sched.schedule(0, lambda: None)
        sched.schedule(0, lambda: None)  # pre-473: "TypeError: unorderable types: function() < function()"t

class SessionTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a PR

    def test_default_serial_consistency_level(self, *_):
        """
        Make sure default_serial_consistency_level passes through to a query message.
        Also make sure Statement.serial_consistency_level overrides the default.

        PR #510
        """
        s = Session(Cluster(protocol_version=4), [])

        # default is None
        self.assertIsNone(s.default_serial_consistency_level)

        sentinel = 1001
        for cl in (None, ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL, sentinel):
            s.default_serial_consistency_level = cl

            # default is passed through
            f = s.execute_async(query='')
            self.assertEqual(f.message.serial_consistency_level, cl)

            # any non-None statement setting takes precedence
            for cl_override in (ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL):
                f = s.execute_async(SimpleStatement(query_string='', serial_consistency_level=cl_override))
                self.assertEqual(s.default_serial_consistency_level, cl)
                self.assertEqual(f.message.serial_consistency_level, cl_override)

class ExecutionProfileTest(unittest.TestCase):

    def _verify_response_future_profile(self, rf, prof):
        self.assertEqual(rf._load_balancer, prof.load_balancing_policy)
        self.assertEqual(rf._retry_policy, prof.retry_policy)
        self.assertEqual(rf.message.consistency_level, prof.consistency_level)
        self.assertEqual(rf.message.serial_consistency_level, prof.serial_consistency_level)
        self.assertEqual(rf.timeout, prof.request_timeout)
        self.assertEqual(rf.row_factory, prof.row_factory)

    def test_default_exec_parameters(self):
        cluster = Cluster()
        self.assertEqual(cluster._config_mode, _ConfigMode.UNCOMMITTED)
        self.assertEqual(cluster.load_balancing_policy.__class__, default_lbp_factory().__class__)
        self.assertEqual(cluster.default_retry_policy.__class__, RetryPolicy)
        session = Session(cluster, hosts=[])
        self.assertEqual(session.default_timeout, 10.0)
        self.assertEqual(session.default_consistency_level, ConsistencyLevel.LOCAL_ONE)
        self.assertEqual(session.default_serial_consistency_level, None)
        self.assertEqual(session.row_factory, named_tuple_factory)

    def test_default_legacy(self):
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy(), default_retry_policy=DowngradingConsistencyRetryPolicy())
        self.assertEqual(cluster._config_mode, _ConfigMode.LEGACY)
        session = Session(cluster, hosts=[])
        session.default_timeout = 3.7
        session.default_consistency_level = ConsistencyLevel.ALL
        session.default_serial_consistency_level = ConsistencyLevel.SERIAL
        rf = session.execute_async("query")
        expected_profile = ExecutionProfile(cluster.load_balancing_policy, cluster.default_retry_policy,
                                            session.default_consistency_level, session.default_serial_consistency_level,
                                            session.default_timeout, session.row_factory)
        self._verify_response_future_profile(rf, expected_profile)

    def test_default_profile(self):
        non_default_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'non-default': non_default_profile})
        session = Session(cluster, hosts=[])

        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        default_profile = cluster.profile_manager.profiles[EXEC_PROFILE_DEFAULT]
        rf = session.execute_async("query")
        self._verify_response_future_profile(rf, default_profile)

        rf = session.execute_async("query", execution_profile='non-default')
        self._verify_response_future_profile(rf, non_default_profile)

    def test_statement_params_override_legacy(self):
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy(), default_retry_policy=DowngradingConsistencyRetryPolicy())
        self.assertEqual(cluster._config_mode, _ConfigMode.LEGACY)
        session = Session(cluster, hosts=[])

        ss = SimpleStatement("query", retry_policy=DowngradingConsistencyRetryPolicy(),
                             consistency_level=ConsistencyLevel.ALL, serial_consistency_level=ConsistencyLevel.SERIAL)
        my_timeout = 1.1234

        self.assertNotEqual(ss.retry_policy.__class__, cluster.default_retry_policy)
        self.assertNotEqual(ss.consistency_level, session.default_consistency_level)
        self.assertNotEqual(ss._serial_consistency_level, session.default_serial_consistency_level)
        self.assertNotEqual(my_timeout, session.default_timeout)

        rf = session.execute_async(ss, timeout=my_timeout)
        expected_profile = ExecutionProfile(load_balancing_policy=cluster.load_balancing_policy, retry_policy=ss.retry_policy,
                                            request_timeout=my_timeout, consistency_level=ss.consistency_level,
                                            serial_consistency_level=ss._serial_consistency_level)
        self._verify_response_future_profile(rf, expected_profile)

    def test_statement_params_override_profile(self):
        non_default_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'non-default': non_default_profile})
        session = Session(cluster, hosts=[])

        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        rf = session.execute_async("query", execution_profile='non-default')

        ss = SimpleStatement("query", retry_policy=DowngradingConsistencyRetryPolicy(),
                             consistency_level=ConsistencyLevel.ALL, serial_consistency_level=ConsistencyLevel.SERIAL)
        my_timeout = 1.1234

        self.assertNotEqual(ss.retry_policy.__class__, rf._load_balancer.__class__)
        self.assertNotEqual(ss.consistency_level, rf.message.consistency_level)
        self.assertNotEqual(ss._serial_consistency_level, rf.message.serial_consistency_level)
        self.assertNotEqual(my_timeout, rf.timeout)

        rf = session.execute_async(ss, timeout=my_timeout, execution_profile='non-default')
        expected_profile = ExecutionProfile(non_default_profile.load_balancing_policy, ss.retry_policy,
                                            ss.consistency_level, ss._serial_consistency_level, my_timeout, non_default_profile.row_factory)
        self._verify_response_future_profile(rf, expected_profile)

    def test_no_profile_with_legacy(self):
        # don't construct with both
        self.assertRaises(ValueError, Cluster, load_balancing_policy=RoundRobinPolicy(), execution_profiles={'a': ExecutionProfile()})
        self.assertRaises(ValueError, Cluster, default_retry_policy=DowngradingConsistencyRetryPolicy(), execution_profiles={'a': ExecutionProfile()})
        self.assertRaises(ValueError, Cluster, load_balancing_policy=RoundRobinPolicy(),
                          default_retry_policy=DowngradingConsistencyRetryPolicy(), execution_profiles={'a': ExecutionProfile()})

        # can't add after
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy())
        self.assertRaises(ValueError, cluster.add_execution_profile, 'name', ExecutionProfile())

        # session settings lock out profiles
        cluster = Cluster()
        session = Session(cluster, hosts=[])
        for attr, value in (('default_timeout', 1),
                            ('default_consistency_level', ConsistencyLevel.ANY),
                            ('default_serial_consistency_level', ConsistencyLevel.SERIAL),
                            ('row_factory', tuple_factory)):
            cluster._config_mode = _ConfigMode.UNCOMMITTED
            setattr(session, attr, value)
            self.assertRaises(ValueError, cluster.add_execution_profile, 'name', ExecutionProfile())

        # don't accept profile
        self.assertRaises(ValueError, session.execute_async, "query", execution_profile='some name here')

    def test_no_legacy_with_profile(self):
        cluster_init = Cluster(execution_profiles={'name': ExecutionProfile()})
        cluster_add = Cluster()
        cluster_add.add_execution_profile('name', ExecutionProfile())
        # for clusters with profiles added either way...
        for c in (cluster_init, cluster_init):
            # don't allow legacy parameters set
            with self.assertRaises(ValueError):
                c.default_retry_policy = RetryPolicy()
                # lbp is not guarded because it would never have worked
                # TODO: guard?
            session = Session(c, hosts=[])
            for attr, value in (('default_timeout', 1),
                                ('default_consistency_level', ConsistencyLevel.ANY),
                                ('default_serial_consistency_level', ConsistencyLevel.SERIAL),
                                ('row_factory', tuple_factory)):
                self.assertRaises(ValueError, setattr, session, attr, value)

    def test_profile_name_value(self):

        internalized_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'by-name': internalized_profile})
        session = Session(cluster, hosts=[])
        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        rf = session.execute_async("query", execution_profile='by-name')
        self._verify_response_future_profile(rf, internalized_profile)

        by_value = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        rf = session.execute_async("query", execution_profile=by_value)
        self._verify_response_future_profile(rf, by_value)
<commit_msg>test legacy lbp rejection<commit_after># Copyright 2013-2016 DataStax, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
try:
    import unittest2 as unittest
except ImportError:
    import unittest  # noqa

from copy import copy
from mock import patch, Mock

from cassandra import ConsistencyLevel, DriverException, Timeout, Unavailable, RequestExecutionException, ReadTimeout, WriteTimeout, CoordinationFailure, ReadFailure, WriteFailure, FunctionFailure, AlreadyExists,\
    InvalidRequest, Unauthorized, AuthenticationFailed, OperationTimedOut, UnsupportedOperation, RequestValidationException, ConfigurationException
from cassandra.cluster import _Scheduler, Session, Cluster, _NOT_SET, default_lbp_factory, \
    ExecutionProfile, _ConfigMode, EXEC_PROFILE_DEFAULT
from cassandra.policies import HostDistance, RetryPolicy, RoundRobinPolicy, DowngradingConsistencyRetryPolicy
from cassandra.query import SimpleStatement, named_tuple_factory, tuple_factory

class ExceptionTypeTest(unittest.TestCase):

    def test_exception_types(self):
        """
        PYTHON-443
        Sanity check to ensure we don't unintentionally change class hierarchy of exception types
        """
        self.assertTrue(issubclass(Unavailable, DriverException))
        self.assertTrue(issubclass(Unavailable, RequestExecutionException))

        self.assertTrue(issubclass(ReadTimeout, DriverException))
        self.assertTrue(issubclass(ReadTimeout, RequestExecutionException))
        self.assertTrue(issubclass(ReadTimeout, Timeout))

        self.assertTrue(issubclass(WriteTimeout, DriverException))
        self.assertTrue(issubclass(WriteTimeout, RequestExecutionException))
        self.assertTrue(issubclass(WriteTimeout, Timeout))

        self.assertTrue(issubclass(CoordinationFailure, DriverException))
        self.assertTrue(issubclass(CoordinationFailure, RequestExecutionException))

        self.assertTrue(issubclass(ReadFailure, DriverException))
        self.assertTrue(issubclass(ReadFailure, RequestExecutionException))
        self.assertTrue(issubclass(ReadFailure, CoordinationFailure))

        self.assertTrue(issubclass(WriteFailure, DriverException))
        self.assertTrue(issubclass(WriteFailure, RequestExecutionException))
        self.assertTrue(issubclass(WriteFailure, CoordinationFailure))

        self.assertTrue(issubclass(FunctionFailure, DriverException))
        self.assertTrue(issubclass(FunctionFailure, RequestExecutionException))

        self.assertTrue(issubclass(RequestValidationException, DriverException))

        self.assertTrue(issubclass(ConfigurationException, DriverException))
        self.assertTrue(issubclass(ConfigurationException, RequestValidationException))

        self.assertTrue(issubclass(AlreadyExists, DriverException))
        self.assertTrue(issubclass(AlreadyExists, RequestValidationException))
        self.assertTrue(issubclass(AlreadyExists, ConfigurationException))

        self.assertTrue(issubclass(InvalidRequest, DriverException))
        self.assertTrue(issubclass(InvalidRequest, RequestValidationException))

        self.assertTrue(issubclass(Unauthorized, DriverException))
        self.assertTrue(issubclass(Unauthorized, RequestValidationException))

        self.assertTrue(issubclass(AuthenticationFailed, DriverException))

        self.assertTrue(issubclass(OperationTimedOut, DriverException))

        self.assertTrue(issubclass(UnsupportedOperation, DriverException))

class ClusterTest(unittest.TestCase):

    def test_invalid_contact_point_types(self):
        with self.assertRaises(ValueError):
            Cluster(contact_points=[None], protocol_version=4, connect_timeout=1)
        with self.assertRaises(TypeError):
            Cluster(contact_points="not a sequence", protocol_version=4, connect_timeout=1)

    def test_requests_in_flight_threshold(self):
        d = HostDistance.LOCAL
        mn = 3
        mx = 5
        c = Cluster(protocol_version=2)
        c.set_min_requests_per_connection(d, mn)
        c.set_max_requests_per_connection(d, mx)
        # min underflow, max, overflow
        for n in (-1, mx, 127):
            self.assertRaises(ValueError, c.set_min_requests_per_connection, d, n)
        # max underflow, under min, overflow
        for n in (0, mn, 128):
            self.assertRaises(ValueError, c.set_max_requests_per_connection, d, n)

class SchedulerTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a ticket

    @patch('time.time', return_value=3)  # always queue at same time
    @patch('cassandra.cluster._Scheduler.run')  # don't actually run the thread
    def test_event_delay_timing(self, *_):
        """
        Schedule something with a time collision to make sure the heap comparison works

        PYTHON-473
        """
        sched = _Scheduler(None)
        sched.schedule(0, lambda: None)
        sched.schedule(0, lambda: None)  # pre-473: "TypeError: unorderable types: function() < function()"t

class SessionTest(unittest.TestCase):
    # TODO: this suite could be expanded; for now just adding a test covering a PR

    def test_default_serial_consistency_level(self, *_):
        """
        Make sure default_serial_consistency_level passes through to a query message.
        Also make sure Statement.serial_consistency_level overrides the default.

        PR #510
        """
        s = Session(Cluster(protocol_version=4), [])

        # default is None
        self.assertIsNone(s.default_serial_consistency_level)

        sentinel = 1001
        for cl in (None, ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL, sentinel):
            s.default_serial_consistency_level = cl

            # default is passed through
            f = s.execute_async(query='')
            self.assertEqual(f.message.serial_consistency_level, cl)

            # any non-None statement setting takes precedence
            for cl_override in (ConsistencyLevel.LOCAL_SERIAL, ConsistencyLevel.SERIAL):
                f = s.execute_async(SimpleStatement(query_string='', serial_consistency_level=cl_override))
                self.assertEqual(s.default_serial_consistency_level, cl)
                self.assertEqual(f.message.serial_consistency_level, cl_override)

class ExecutionProfileTest(unittest.TestCase):

    def _verify_response_future_profile(self, rf, prof):
        self.assertEqual(rf._load_balancer, prof.load_balancing_policy)
        self.assertEqual(rf._retry_policy, prof.retry_policy)
        self.assertEqual(rf.message.consistency_level, prof.consistency_level)
        self.assertEqual(rf.message.serial_consistency_level, prof.serial_consistency_level)
        self.assertEqual(rf.timeout, prof.request_timeout)
        self.assertEqual(rf.row_factory, prof.row_factory)

    def test_default_exec_parameters(self):
        cluster = Cluster()
        self.assertEqual(cluster._config_mode, _ConfigMode.UNCOMMITTED)
        self.assertEqual(cluster.load_balancing_policy.__class__, default_lbp_factory().__class__)
        self.assertEqual(cluster.default_retry_policy.__class__, RetryPolicy)
        session = Session(cluster, hosts=[])
        self.assertEqual(session.default_timeout, 10.0)
        self.assertEqual(session.default_consistency_level, ConsistencyLevel.LOCAL_ONE)
        self.assertEqual(session.default_serial_consistency_level, None)
        self.assertEqual(session.row_factory, named_tuple_factory)

    def test_default_legacy(self):
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy(), default_retry_policy=DowngradingConsistencyRetryPolicy())
        self.assertEqual(cluster._config_mode, _ConfigMode.LEGACY)
        session = Session(cluster, hosts=[])
        session.default_timeout = 3.7
        session.default_consistency_level = ConsistencyLevel.ALL
        session.default_serial_consistency_level = ConsistencyLevel.SERIAL
        rf = session.execute_async("query")
        expected_profile = ExecutionProfile(cluster.load_balancing_policy, cluster.default_retry_policy,
                                            session.default_consistency_level, session.default_serial_consistency_level,
                                            session.default_timeout, session.row_factory)
        self._verify_response_future_profile(rf, expected_profile)

    def test_default_profile(self):
        non_default_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'non-default': non_default_profile})
        session = Session(cluster, hosts=[])

        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        default_profile = cluster.profile_manager.profiles[EXEC_PROFILE_DEFAULT]
        rf = session.execute_async("query")
        self._verify_response_future_profile(rf, default_profile)

        rf = session.execute_async("query", execution_profile='non-default')
        self._verify_response_future_profile(rf, non_default_profile)

    def test_statement_params_override_legacy(self):
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy(), default_retry_policy=DowngradingConsistencyRetryPolicy())
        self.assertEqual(cluster._config_mode, _ConfigMode.LEGACY)
        session = Session(cluster, hosts=[])

        ss = SimpleStatement("query", retry_policy=DowngradingConsistencyRetryPolicy(),
                             consistency_level=ConsistencyLevel.ALL, serial_consistency_level=ConsistencyLevel.SERIAL)
        my_timeout = 1.1234

        self.assertNotEqual(ss.retry_policy.__class__, cluster.default_retry_policy)
        self.assertNotEqual(ss.consistency_level, session.default_consistency_level)
        self.assertNotEqual(ss._serial_consistency_level, session.default_serial_consistency_level)
        self.assertNotEqual(my_timeout, session.default_timeout)

        rf = session.execute_async(ss, timeout=my_timeout)
        expected_profile = ExecutionProfile(load_balancing_policy=cluster.load_balancing_policy, retry_policy=ss.retry_policy,
                                            request_timeout=my_timeout, consistency_level=ss.consistency_level,
                                            serial_consistency_level=ss._serial_consistency_level)
        self._verify_response_future_profile(rf, expected_profile)

    def test_statement_params_override_profile(self):
        non_default_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'non-default': non_default_profile})
        session = Session(cluster, hosts=[])

        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        rf = session.execute_async("query", execution_profile='non-default')

        ss = SimpleStatement("query", retry_policy=DowngradingConsistencyRetryPolicy(),
                             consistency_level=ConsistencyLevel.ALL, serial_consistency_level=ConsistencyLevel.SERIAL)
        my_timeout = 1.1234

        self.assertNotEqual(ss.retry_policy.__class__, rf._load_balancer.__class__)
        self.assertNotEqual(ss.consistency_level, rf.message.consistency_level)
        self.assertNotEqual(ss._serial_consistency_level, rf.message.serial_consistency_level)
        self.assertNotEqual(my_timeout, rf.timeout)

        rf = session.execute_async(ss, timeout=my_timeout, execution_profile='non-default')
        expected_profile = ExecutionProfile(non_default_profile.load_balancing_policy, ss.retry_policy,
                                            ss.consistency_level, ss._serial_consistency_level, my_timeout, non_default_profile.row_factory)
        self._verify_response_future_profile(rf, expected_profile)

    def test_no_profile_with_legacy(self):
        # don't construct with both
        self.assertRaises(ValueError, Cluster, load_balancing_policy=RoundRobinPolicy(), execution_profiles={'a': ExecutionProfile()})
        self.assertRaises(ValueError, Cluster, default_retry_policy=DowngradingConsistencyRetryPolicy(), execution_profiles={'a': ExecutionProfile()})
        self.assertRaises(ValueError, Cluster, load_balancing_policy=RoundRobinPolicy(),
                          default_retry_policy=DowngradingConsistencyRetryPolicy(), execution_profiles={'a': ExecutionProfile()})

        # can't add after
        cluster = Cluster(load_balancing_policy=RoundRobinPolicy())
        self.assertRaises(ValueError, cluster.add_execution_profile, 'name', ExecutionProfile())

        # session settings lock out profiles
        cluster = Cluster()
        session = Session(cluster, hosts=[])
        for attr, value in (('default_timeout', 1),
                            ('default_consistency_level', ConsistencyLevel.ANY),
                            ('default_serial_consistency_level', ConsistencyLevel.SERIAL),
                            ('row_factory', tuple_factory)):
            cluster._config_mode = _ConfigMode.UNCOMMITTED
            setattr(session, attr, value)
            self.assertRaises(ValueError, cluster.add_execution_profile, 'name', ExecutionProfile())

        # don't accept profile
        self.assertRaises(ValueError, session.execute_async, "query", execution_profile='some name here')

    def test_no_legacy_with_profile(self):
        cluster_init = Cluster(execution_profiles={'name': ExecutionProfile()})
        cluster_add = Cluster()
        cluster_add.add_execution_profile('name', ExecutionProfile())
        # for clusters with profiles added either way...
        for cluster in (cluster_init, cluster_init):
            # don't allow legacy parameters set
            for attr, value in (('default_retry_policy', RetryPolicy()),
                                ('load_balancing_policy', default_lbp_factory())):
                self.assertRaises(ValueError, setattr, cluster, attr, value)
            session = Session(cluster, hosts=[])
            for attr, value in (('default_timeout', 1),
                                ('default_consistency_level', ConsistencyLevel.ANY),
                                ('default_serial_consistency_level', ConsistencyLevel.SERIAL),
                                ('row_factory', tuple_factory)):
                self.assertRaises(ValueError, setattr, session, attr, value)

    def test_profile_name_value(self):

        internalized_profile = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        cluster = Cluster(execution_profiles={'by-name': internalized_profile})
        session = Session(cluster, hosts=[])
        self.assertEqual(cluster._config_mode, _ConfigMode.PROFILES)

        rf = session.execute_async("query", execution_profile='by-name')
        self._verify_response_future_profile(rf, internalized_profile)

        by_value = ExecutionProfile(RoundRobinPolicy(), *[object() for _ in range(5)])
        rf = session.execute_async("query", execution_profile=by_value)
        self._verify_response_future_profile(rf, by_value)
<|endoftext|>


<commit_before>import pytest
from hijriconverter import convert

@pytest.fixture()
def hijri():
    return convert.Hijri(1410, 8, 13)

def test_ignore_validation_for_hijri_object():
    try:
        convert.Hijri(1410, 8, 30, 'lunar', validate=False)
    except TypeError or ValueError:
        pytest.fail()

def test_representation_of_hijri_object(hijri):
    assert hijri.__repr__() == 'Hijri(1410, 8, 13, lunar)'

def test_isoformat(hijri):
    assert hijri.isoformat() == '1410-08-13'

def test_year_number(hijri):
    assert hijri.year == 1410

def test_month_number(hijri):
    assert hijri.month == 8

def test_day_number(hijri):
    assert hijri.day == 13

def test_datetuple(hijri):
    assert hijri.datetuple() == (1410, 8, 13)

def test_month_days(hijri):
    assert hijri.month_days() == 29

def test_month_name(hijri):
    assert hijri.month_name() == 'Sha’ban'

def test_weekday(hijri):
    assert hijri.weekday() == 5

def test_iso_weekday(hijri):
    assert hijri.isoweekday() == 6

def test_day_name(hijri):
    assert hijri.day_name() == 'Saturday'

def test_convert_hijri_date_to_gregorian(hijri):
    assert hijri.to_gregorian().timetuple()[:3] == (1990, 3, 10)

@pytest.fixture()
def gregorian():
    return convert.Gregorian(1990, 3, 10)

def test_representation_of_gregorian_object(gregorian):
    assert repr(gregorian) == 'Gregorian(1990, 3, 10)'

def test_convert_gregorian_date_to_hijri(gregorian):
    assert gregorian.to_hijri().datetuple() == (1410, 8, 13)
<commit_msg>fix a typo<commit_after>import pytest
from hijriconverter import convert

@pytest.fixture()
def hijri():
    return convert.Hijri(1410, 8, 13)

def test_ignore_validation_for_hijri_object():
    try:
        convert.Hijri(1410, 8, 30, 'lunar', validated=True)
    except TypeError or ValueError:
        pytest.fail()

def test_representation_of_hijri_object(hijri):
    assert hijri.__repr__() == 'Hijri(1410, 8, 13, lunar)'

def test_isoformat(hijri):
    assert hijri.isoformat() == '1410-08-13'

def test_year_number(hijri):
    assert hijri.year == 1410

def test_month_number(hijri):
    assert hijri.month == 8

def test_day_number(hijri):
    assert hijri.day == 13

def test_datetuple(hijri):
    assert hijri.datetuple() == (1410, 8, 13)

def test_month_days(hijri):
    assert hijri.month_days() == 29

def test_month_name(hijri):
    assert hijri.month_name() == 'Sha’ban'

def test_weekday(hijri):
    assert hijri.weekday() == 5

def test_iso_weekday(hijri):
    assert hijri.isoweekday() == 6

def test_day_name(hijri):
    assert hijri.day_name() == 'Saturday'

def test_convert_hijri_date_to_gregorian(hijri):
    assert hijri.to_gregorian().timetuple()[:3] == (1990, 3, 10)

@pytest.fixture()
def gregorian():
    return convert.Gregorian(1990, 3, 10)

def test_representation_of_gregorian_object(gregorian):
    assert repr(gregorian) == 'Gregorian(1990, 3, 10)'

def test_convert_gregorian_date_to_hijri(gregorian):
    assert gregorian.to_hijri().datetuple() == (1410, 8, 13)
<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
This script is designed to update the expected test output.
It assumes that you have already run nosetests and ran the entire test suite.
By doing so, the output has been generated under the given outputs directory.
and that is what will be used to generate the new expected output under `tests/data/experiments`.

#############################################################################################
# IMPORTANT: DO NOT RUN THIS SCRIPT BEFORE RUNNING THE TEST SUITE OR IT WILL BE DISASTROUS. #
#############################################################################################

The scripts works as as follows. For each experiment test:
- The script locates the output under the given outputs directory.
- New and changed files under `test_outputs` are copied over to the expected test output location.
- Old files in the expected test output that DO NOT exist under `test_outputs` are deleted.
- Files that are already in the expected test output and have not changed are left alone.

The script prints a log detailing the changes made for each experiment test.

:author: Anastassia Loukina
:author: Nitin Madnani
:author: Jeremy Biggs
:date: Feburary 2018
"""

import argparse
import re
import sys

from ast import literal_eval as eval
from filecmp import dircmp
from importlib.machinery import SourceFileLoader
from inspect import getmembers, getsourcelines, isfunction
from os import remove
from os.path import dirname, join
from pathlib import Path
from shutil import copyfile

_MY_PATH = dirname(__file__)

def is_skll_excluded_file(filename):

    stem = Path(filename).stem
    suffix = Path(filename).suffix
    return suffix == '.model' or \
        suffix == '.npy'or \
        stem.endswith('_postprocessing_params') or \
        stem.endswith('_eval') or \
        stem.endswith('_eval_short') or \
        stem.endswith('_confMatrix') or \
        stem.endswith('_pred_train') or \
        stem.endswith('_pred_processed') or \
        stem.endswith('_score_dist')

def update_output(source, outputs_dir, skll=False):

    # locate the updated outputs for the experiment under the given
    # outputs directory and also locate the existing experiment outputs
    updated_output_path = outputs_dir / source / "output"
    existing_output_path = Path(_MY_PATH) / "data" / "experiments" / source / "output"

    # get a comparison betwen the two directories
    dir_comparison = dircmp(updated_output_path, existing_output_path)

    # first delete the files that only exist in the existing output directory
    # since those are likely old files from old versions that we do not need
    existing_output_only_files = dir_comparison.right_only
    for file in existing_output_only_files:
        remove(existing_output_path / file)

    # Next find all the files from the updated output path that are either new
    # or changed compared to the existing path. From these we want to exclude
    # config JSON files as well as evaluation/prediction/model files for SKLL
    new_or_changed_files = set(dir_comparison.left_only) - set(dir_comparison.same_files)
    new_or_changed_files = [f for f in new_or_changed_files if not f.endswith('_rsmtool.json') and not f.endswith('_rsmeval.json')]
    if skll:
        new_or_changed_files = [f for f in new_or_changed_files if not is_skll_excluded_file(f)]

    for file in new_or_changed_files:
        copyfile(updated_output_path / file, existing_output_path / file)

    return list(existing_output_only_files), list(new_or_changed_files)

def main():
    # set up an argument parser
    parser = argparse.ArgumentParser(prog='update_test_files.py')
    parser.add_argument('outputs_dir', help="The path to the directory containing the updated test outputs")

    # parse given command line arguments
    args = parser.parse_args()

    # print out a reminder that the user should have run the test suite
    run_test_suite = input('Have you already run the whole test suite? (y/n): ')
    if run_test_suite == 'n':
        print('Please run the whole test suite using '
              '`nosetests --nologcapture` before running this script.')
        sys.exit(0)
    elif run_test_suite != 'y':
        print('Invalid answer. Exiting.')
        sys.exit(1)

    # import the test_experiment.py test module using SourceFileLoader
    # so that it could also be used with tests in RSMExtra provided
    # they folllow the same structure.
    # see http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
    # for Python 3.5 solution
    for test_suffix in ['rsmtool_1', 'rsmtool_2', 'rsmtool_3', 'rsmtool_4',
                        'rsmeval', 'rsmpredict', 'rsmsummarize']:
        test_module = SourceFileLoader('test_experiment', join(_MY_PATH, 'test_experiment_{}.py'.format(test_suffix))).load_module()

        # iterate over all the members and focus on only the experiment functions
        # and try to get the source and the experiment ID since that's what we
        # need to update the test files
        for member_name, member_object in getmembers(test_module):
            if isfunction(member_object) and member_name.startswith('test_run_experiment'):
                function = member_object

                # first we check if it's the parameterized function and if so
                # we can easily get the source and the experiment ID from
                # the parameter list
                if member_name.endswith('parameterized'):
                    for param in function.parameterized_input:
                        source = param.args[0]
                        skll = param.kwargs.get('skll', False)

                        deleted, updated = update_output(source,
                                                         Path(args.outputs_dir),
                                                         skll=skll)
                        if len(deleted) > 0 or len(updated) > 0:
                            print('{}: '.format(source))
                            print('  - {} deleted: {}'.format(len(deleted), deleted))
                            print('  - {} added/updated: {}'.format(len(updated), updated))

                # if it's another function, then we actually inspect the source
                # to get the source and experiment_id
                else:
                    function_code_lines = getsourcelines(function)
                    experiment_id_line = [line for line in function_code_lines[0]
                                          if re.search(r'experiment_id = ', line)]

                    # if there was no experiment ID specified, it was either a
                    # a decorated test function (which means it wouldn't have an 'output'
                    # directory anyway) or it was a compare or prediction test function
                    # which are not a problem for various reasons.
                    if experiment_id_line:

                        # get the name of the source directory
                        source_line = [line for line in function_code_lines[0]
                                       if re.search(r'source = ', line)]
                        source = eval(source_line[0].strip().split(' = ')[1])
                        deleted, updated = update_output(source, Path(args.outputs_dir))
                        if len(deleted) > 0 or len(updated) > 0:
                            print('{}: '.format(source))
                            print('  - {} deleted: {}'.format(len(deleted), deleted))
                            print('  - {} added/updated: {}'.format(len(updated), updated))

if __name__ == '__main__':
    main()
<commit_msg>Fix bug in finding changed files.<commit_after>#!/usr/bin/env python

"""
This script is designed to update the expected test output.
It assumes that you have already run nosetests and ran the entire test suite.
By doing so, the output has been generated under the given outputs directory.
and that is what will be used to generate the new expected output under `tests/data/experiments`.

#############################################################################################
# IMPORTANT: DO NOT RUN THIS SCRIPT BEFORE RUNNING THE TEST SUITE OR IT WILL BE DISASTROUS. #
#############################################################################################

The scripts works as as follows. For each experiment test:
- The script locates the output under the given outputs directory.
- New and changed files under `test_outputs` are copied over to the expected test output location.
- Old files in the expected test output that DO NOT exist under `test_outputs` are deleted.
- Files that are already in the expected test output and have not changed are left alone.

The script prints a log detailing the changes made for each experiment test.

:author: Anastassia Loukina
:author: Nitin Madnani
:author: Jeremy Biggs
:date: Feburary 2018
"""

import argparse
import re
import sys

from ast import literal_eval as eval
from filecmp import clear_cache, dircmp
from importlib.machinery import SourceFileLoader
from inspect import getmembers, getsourcelines, isfunction
from os import remove
from os.path import dirname, join
from pathlib import Path
from shutil import copyfile

_MY_PATH = dirname(__file__)

def is_skll_excluded_file(filename):

    stem = Path(filename).stem
    suffix = Path(filename).suffix
    return suffix == '.model' or \
        suffix == '.npy'or \
        stem.endswith('_postprocessing_params') or \
        stem.endswith('_eval') or \
        stem.endswith('_eval_short') or \
        stem.endswith('_confMatrix') or \
        stem.endswith('_pred_train') or \
        stem.endswith('_pred_processed') or \
        stem.endswith('_score_dist')

def update_output(source, outputs_dir, skll=False):

    # locate the updated outputs for the experiment under the given
    # outputs directory and also locate the existing experiment outputs
    updated_output_path = outputs_dir / source / "output"
    existing_output_path = Path(_MY_PATH) / "data" / "experiments" / source / "output"

    # make sure both the directories exist
    try:
        assert updated_output_path.exists()
    except AssertionError:
        sys.stderr.write("\nError: the given directory with updated "
                         "test outputs does not contain the outputs for "
                         "the \"{}\" experiment.\n".format(source))
        sys.exit(1)

    try:
        assert existing_output_path.exists()
    except AssertionError:
        sys.stderr.write("\nError: the test directory for \"{}\" does not contain "
                         "an output directory. Please create one.\n".format(source))
        sys.exit(1)

    # get a comparison betwen the two directories
    dir_comparison = dircmp(updated_output_path, existing_output_path)

    # first delete the files that only exist in the existing output directory
    # since those are likely old files from old versions that we do not need
    existing_output_only_files = dir_comparison.right_only
    for file in existing_output_only_files:
        remove(existing_output_path / file)

    # Next find all the files from the updated output path that are either new
    # or changed compared to the existing path. From these we want to exclude
    # config JSON files as well as evaluation/prediction/model files for SKLL
    new_or_changed_files = set(dir_comparison.left_only) | set(dir_comparison.diff_files)
    new_or_changed_files = [f for f in new_or_changed_files if not f.endswith('_rsmtool.json') and not f.endswith('_rsmeval.json')]
    if skll:
        new_or_changed_files = [f for f in new_or_changed_files if not is_skll_excluded_file(f)]

    for file in new_or_changed_files:
        copyfile(updated_output_path / file, existing_output_path / file)

    return sorted(existing_output_only_files), sorted(new_or_changed_files)

def main():
    # set up an argument parser
    parser = argparse.ArgumentParser(prog='update_test_files.py')
    parser.add_argument('outputs_dir', help="The path to the directory containing the updated test outputs")

    # parse given command line arguments
    args = parser.parse_args()

    # invalidate the filecmp cache so that it starts fresh
    clear_cache()

    # print out a reminder that the user should have run the test suite
    run_test_suite = input('Have you already run the whole test suite? (y/n): ')
    if run_test_suite == 'n':
        print('Please run the whole test suite using '
              '`nosetests --nologcapture` before running this script.')
        sys.exit(0)
    elif run_test_suite != 'y':
        print('Invalid answer. Exiting.')
        sys.exit(1)

    # import the test_experiment.py test module using SourceFileLoader
    # so that it could also be used with tests in RSMExtra provided
    # they folllow the same structure.
    # see http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path
    # for Python 3.5 solution
    for test_suffix in ['rsmtool_1', 'rsmtool_2', 'rsmtool_3', 'rsmtool_4',
                        'rsmeval', 'rsmpredict', 'rsmsummarize']:
        test_module = SourceFileLoader('test_experiment', join(_MY_PATH, 'test_experiment_{}.py'.format(test_suffix))).load_module()

        # iterate over all the members and focus on only the experiment functions
        # and try to get the source and the experiment ID since that's what we
        # need to update the test files
        for member_name, member_object in getmembers(test_module):
            if isfunction(member_object) and member_name.startswith('test_run_experiment'):
                function = member_object

                # first we check if it's the parameterized function and if so
                # we can easily get the source and the experiment ID from
                # the parameter list
                if member_name.endswith('parameterized'):
                    for param in function.parameterized_input:
                        source = param.args[0]
                        skll = param.kwargs.get('skll', False)

                        deleted, updated = update_output(source,
                                                         Path(args.outputs_dir),
                                                         skll=skll)
                        if len(deleted) > 0 or len(updated) > 0:
                            print('{}: '.format(source))
                            print('  - {} deleted: {}'.format(len(deleted), deleted))
                            print('  - {} added/updated: {}'.format(len(updated), updated))

                # if it's another function, then we actually inspect the source
                # to get the source and experiment_id
                else:
                    function_code_lines = getsourcelines(function)
                    experiment_id_line = [line for line in function_code_lines[0]
                                          if re.search(r'experiment_id = ', line)]

                    # if there was no experiment ID specified, it was either a
                    # a decorated test function (which means it wouldn't have an 'output'
                    # directory anyway) or it was a compare or prediction test function
                    # which are not a problem for various reasons.
                    if experiment_id_line:

                        # get the name of the source directory
                        source_line = [line for line in function_code_lines[0]
                                       if re.search(r'source = ', line)]
                        source = eval(source_line[0].strip().split(' = ')[1])
                        deleted, updated = update_output(source, Path(args.outputs_dir))
                        if len(deleted) > 0 or len(updated) > 0:
                            print('{}: '.format(source))
                            print('  - {} deleted: {}'.format(len(deleted), deleted))
                            print('  - {} added/updated: {}'.format(len(updated), updated))

if __name__ == '__main__':
    main()
<|endoftext|>


<commit_before>import logging
import time
import socket
import struct
import codecs
import random
import datetime
from circuits import Component, Event, Timer, Worker, task

from solar_circuit.libs.pyModbusTCP.client import ModbusClient
from solar_circuit.libs import prettytable
from solar_circuit.utility.helpers import stringify_reg, power_error
from solar_circuit.utility import formats
from solar_circuit.sample_database import store_sample
from . import sample, sample_success

TCP_DEV_NAMES ={}

def register_device_type(cls):
	global TCP_DEV_NAMES
	TCP_DEV_NAMES[cls.__name__] = cls
	return cls

@register_device_type
class ModbusTCPDevice(Component):
	MODBUS_PORT = 502
	DEFAULT_INTERVAL = 60 * 15
	PREFIX = 'XXXX'
	def __init__(self, ip):
		super(ModbusTCPDevice, self).__init__()
		self.sample_timer = None
		self.ip = ip
		self.sn = ip
		self.registers = []
		self.conn = None
		self._set_channel()

	def _set_channel(self):
		self.channel = self.__class__.__name__ + str(id(self))

	def started(self, *args):
		self.conn = ModbusClient(host=self.ip, port=self.MODBUS_PORT,
								 auto_open=True, auto_close=True)
		self.fire(sample(), self)
		self.sample_timer = Timer(self.DEFAULT_INTERVAL + random.uniform(0,1), sample(),
								  self, persist=True).register(self)
		return

	def update_interval(self, interval, stray=1):
		if self.sample_timer is not None:
			self.sample_timer.reset(interval + random.uniform(0,stray))
			return True
		return False

	def sample_success(self, addr, regs):
		tab = prettytable.PrettyTable()
		# tab.field_names = [str(x) for x in xrange(0xf)]
		num_rows = (len(regs) / 0x10) + 1
		for x in xrange(0x10):
			column = [stringify_reg([_]) for _ in regs[x::0x10]]
			column += [''] * (num_rows - len(column))
			tab.add_column(str(x), column)
		logging.debug("Sample from %s address %s:\n%s", addr, self.sn, tab.get_string())

	def sample(self):
		logging.debug("%s Spawning sample task", self.sn)
		self.fire(task(self._sample), "sample_worker")

	def _sample(self):
		logging.debug("Sampling from %s", self.sn)
		start = time.time()
		for t in self.registers:
			try:
				regs = self.conn.read_holding_registers(t[0], t[1])
			except Exception as e:
				logging.error(e)
			else:
				if regs is None:
					logging.warn("%s sent empty sample", self.sn)
				else:
					self.fire(sample_success(t[0], regs), self)
		time_d = time.time() - start
		logging.debug("Sampling %s took %s", self.sn, time_d)
		return

	def get_dev_id(self):
		return "{}-{}".format(self.PREFIX, self.sn)

@register_device_type
class Shark100(ModbusTCPDevice):
	PREFIX = 'SHRK'
	ERROR_LIMIT = 1.0
	def __init__(self, ip):
		super(Shark100, self).__init__(ip)
		self.registers = [(0x0000, 47),
						  (0x0383, 6),
						  (0x03E7, 30)]
						  # (0x044B, 18),
						  # (0x07CF, 20),
						  # (0x0BB7, 34),
						  # (0x0C1B, 34),
						  # (0x1003, 6),
						  # (0x1387, 4)]

	def started(self, *args):
		super(Shark100, self).started(args)
		self.update_interval(5)

	def sample_success(self, addr, regs):
		sample = {}
		timestamp = datetime.datetime.utcnow()
		if addr == 0x0000:
			# logging.info("Name: %s", formats.modbus_string(regs[0:8]))
			self.sn = formats.modbus_string(regs[8:16]).strip(" ")
			# logging.info("Type: %s", formats.bitfield(regs[16:17]))
			# logging.info("Firmware: %s", formats.modbus_string(regs[17:19]))
			# logging.info("Map Version: %s", formats.uint16(regs[19:20]))
			# logging.info("Meter Configuration: %s", formats.bitfield(regs[20:21]))
			# logging.info("ASIC Version: %s", formats.uint16(regs[21:22]))
		elif addr == 0x0383:
			sample["PowerFast"] = formats.float32(regs[0:2])
			sample["VARFast"] = formats.float32(regs[2:4])
			sample["VAFast"] = formats.float32(regs[4:6])
		elif addr == 0x03E7:
			sample["VoltsAN"] = formats.float32(regs[0:2])
			sample["VoltsBN"] = formats.float32(regs[2:4])
			sample["VoltsCN"] = formats.float32(regs[4:6])
			sample["VoltsAB"] = formats.float32(regs[6:8])
			sample["VoltsBC"] = formats.float32(regs[8:10])
			sample["VoltsCA"] = formats.float32(regs[10:12])
			sample["AmpsA"] = formats.float32(regs[12:14])
			sample["AmpsB"] = formats.float32(regs[14:16])
			sample["AmpsC"] = formats.float32(regs[16:18])
			sample["Power"] = formats.float32(regs[18:20])
			sample["VAR"] = formats.float32(regs[20:22])
			sample["VA"] = formats.float32(regs[22:24])
			sample["PowerFactor"] = formats.float32(regs[24:26])
			sample["Frequency"] = formats.float32(regs[26:28])
			sample["AmpsN"] = formats.float32(regs[28:30])
			error = power_error(sample["Power"], sample["VA"], sample["PowerFactor"])
			if error > self.ERROR_LIMIT:
				logging.error("%s error too high: %f", self.get_dev_id(), error)
				return

		self.fire(store_sample(self.get_dev_id(), timestamp, sample))
<commit_msg>Watching sampling time to completion<commit_after>import logging
import time
import socket
import struct
import codecs
import random
import datetime
from circuits import Component, Event, Timer, Worker, task

from solar_circuit.libs.pyModbusTCP.client import ModbusClient
from solar_circuit.libs import prettytable
from solar_circuit.utility.helpers import stringify_reg, power_error
from solar_circuit.utility import formats
from solar_circuit.sample_database import store_sample
from . import sample, sample_success

TCP_DEV_NAMES ={}

def register_device_type(cls):
	global TCP_DEV_NAMES
	TCP_DEV_NAMES[cls.__name__] = cls
	return cls

@register_device_type
class ModbusTCPDevice(Component):
	MODBUS_PORT = 502
	DEFAULT_INTERVAL = 60 * 15
	PREFIX = 'XXXX'
	def __init__(self, ip):
		super(ModbusTCPDevice, self).__init__()
		self.sample_timer = None
		self.ip = ip
		self.sn = ip
		self.registers = []
		self.conn = None
		self._set_channel()
		self.queued_time = time.time()
		self.time_d = 0

	def _set_channel(self):
		self.channel = self.__class__.__name__ + str(id(self))

	def started(self, *args):
		self.conn = ModbusClient(host=self.ip, port=self.MODBUS_PORT,
								 auto_open=True, auto_close=True)
		self.fire(sample(), self)
		self.sample_timer = Timer(self.DEFAULT_INTERVAL + random.uniform(0,1), sample(),
								  self, persist=True).register(self)
		return

	def update_interval(self, interval, stray=1):
		if self.sample_timer is not None:
			self.sample_timer.reset(interval + random.uniform(0,stray))
			return True
		return False

	def sample_success(self, addr, regs):
		tab = prettytable.PrettyTable()
		# tab.field_names = [str(x) for x in xrange(0xf)]
		num_rows = (len(regs) / 0x10) + 1
		for x in xrange(0x10):
			column = [stringify_reg([_]) for _ in regs[x::0x10]]
			column += [''] * (num_rows - len(column))
			tab.add_column(str(x), column)
		logging.debug("Sample from %s address %s:\n%s", addr, self.sn, tab.get_string())

	def sample(self):
		logging.debug("%s Spawning sample task", self.sn)
		self.queued_time = time.time()
		self.fire(task(self._sample), "sample_worker")

	def _sample(self):
		logging.debug("Sampling from %s", self.sn)
		start = time.time()
		for t in self.registers:
			try:
				regs = self.conn.read_holding_registers(t[0], t[1])
			except Exception as e:
				logging.error(e)
			else:
				if regs is None:
					logging.warn("%s sent empty sample", self.sn)
				else:
					self.fire(sample_success(t[0], regs), self)
		self.time_d = time.time() - self.queued_time
		logging.debug("Sampling %s took %s", self.sn, self.time_d)
		if self.time_d > self.sample_timer.interval:
			logging.warn("Sampling %s took %s! (> %s)", self.sn, self.time_d, self.sample_timer.interval)
		return

	def get_dev_id(self):
		return "{}-{}".format(self.PREFIX, self.sn)

@register_device_type
class Shark100(ModbusTCPDevice):
	PREFIX = 'SHRK'
	ERROR_LIMIT = 1.0
	def __init__(self, ip):
		super(Shark100, self).__init__(ip)
		self.registers = [(0x0000, 47),
						  (0x0383, 6),
						  (0x03E7, 30)]
						  # (0x044B, 18),
						  # (0x07CF, 20),
						  # (0x0BB7, 34),
						  # (0x0C1B, 34),
						  # (0x1003, 6),
						  # (0x1387, 4)]

	def started(self, *args):
		super(Shark100, self).started(args)
		self.update_interval(5)

	def sample_success(self, addr, regs):
		sample = {}
		timestamp = datetime.datetime.utcnow()
		if addr == 0x0000:
			# logging.info("Name: %s", formats.modbus_string(regs[0:8]))
			self.sn = formats.modbus_string(regs[8:16]).strip(" ")
			# logging.info("Type: %s", formats.bitfield(regs[16:17]))
			# logging.info("Firmware: %s", formats.modbus_string(regs[17:19]))
			# logging.info("Map Version: %s", formats.uint16(regs[19:20]))
			# logging.info("Meter Configuration: %s", formats.bitfield(regs[20:21]))
			# logging.info("ASIC Version: %s", formats.uint16(regs[21:22]))
		elif addr == 0x0383:
			sample["PowerFast"] = formats.float32(regs[0:2])
			sample["VARFast"] = formats.float32(regs[2:4])
			sample["VAFast"] = formats.float32(regs[4:6])
		elif addr == 0x03E7:
			sample["VoltsAN"] = formats.float32(regs[0:2])
			sample["VoltsBN"] = formats.float32(regs[2:4])
			sample["VoltsCN"] = formats.float32(regs[4:6])
			sample["VoltsAB"] = formats.float32(regs[6:8])
			sample["VoltsBC"] = formats.float32(regs[8:10])
			sample["VoltsCA"] = formats.float32(regs[10:12])
			sample["AmpsA"] = formats.float32(regs[12:14])
			sample["AmpsB"] = formats.float32(regs[14:16])
			sample["AmpsC"] = formats.float32(regs[16:18])
			sample["Power"] = formats.float32(regs[18:20])
			sample["VAR"] = formats.float32(regs[20:22])
			sample["VA"] = formats.float32(regs[22:24])
			sample["PowerFactor"] = formats.float32(regs[24:26])
			sample["Frequency"] = formats.float32(regs[26:28])
			sample["AmpsN"] = formats.float32(regs[28:30])
			error = power_error(sample["Power"], sample["VA"], sample["PowerFactor"])
			if error > self.ERROR_LIMIT:
				logging.error("%s error too high: %f", self.get_dev_id(), error)
				return

		self.fire(store_sample(self.get_dev_id(), timestamp, sample))
<|endoftext|>