<commit_before>import os
import shutil

from hestia.paths import check_or_create_path, create_path, delete_path
from hestia.service_interface import InvalidService, Service
from marshmallow import ValidationError
from polystores import StoreManager
from rhea import RheaError

from libs.paths.experiment_jobs import create_experiment_job_path
from libs.paths.experiments import create_experiment_path
from libs.paths.jobs import create_job_path
from polyaxon.config_manager import config
from stores.exceptions import VolumeNotFoundError
from stores.schemas.store import StoreConfig
from stores.schemas.volume import VolumeConfig
from stores.store_secrets import get_store_secret_for_persistence, get_store_secret_from_definition

class StoresService(Service):
    __all__ = (
        'get_data_paths',
        'get_data_path',
        'delete_data_path',
        'get_outputs_path',
        'delete_outputs_path',
        'get_logs_path',
        'delete_logs_path',
        'get_outputs_store',
        'get_logs_store',
        'is_bucket_logs_persistence',
        'get_experiment_group_outputs_path',
        'get_experiment_group_logs_path',
        'get_experiment_job_logs_path',
        'upload_experiment_job_logs',
        'get_experiment_outputs_path',
        'get_experiment_logs_path',
        'get_job_outputs_path',
        'get_job_logs_path',
        'upload_job_logs',
        'get_project_outputs_path',
        'get_project_logs_path',
        'create_experiment_logs_path',
        'create_experiment_outputs_path',
        'create_experiment_job_logs_path',
        'upload_experiment_logs',
        'copy_experiment_outputs',
        'create_job_logs_path',
        'create_job_outputs_path'
    )

    @staticmethod
    def get_data_paths(persistences):
        import conf
        from stores.validators import validate_persistence_data

        persistence_data = validate_persistence_data(persistence_data=persistences)
        persistence_paths = {}
        for persistence in persistence_data:
            if persistence not in conf.get('PERSISTENCE_DATA'):
                raise VolumeNotFoundError(
                    'Data volume with name `{}` was defined in specification, '
                    'but was not found'.format(persistence))
            persistence_type_condition = (
                'mountPath' not in conf.get('PERSISTENCE_DATA')[persistence] and
                'bucket' not in conf.get('PERSISTENCE_DATA')[persistence]
            )
            if persistence_type_condition:
                raise VolumeNotFoundError(
                    'Data volume with name `{}` '
                    'does not define a mountPath or bucket.'.format(persistence))

            persistence_paths[persistence] = (
                conf.get('PERSISTENCE_DATA')[persistence].get('mountPath') or
                conf.get('PERSISTENCE_DATA')[persistence].get('bucket'))

        return persistence_paths

    @classmethod
    def get_data_path(cls, persistence):
        persistences = [persistence] if persistence else None
        paths = cls.get_data_paths(persistences=persistences)
        return list(paths.values())[0]

    @classmethod
    def delete_data_path(cls, subpath, persistence):
        data_path = cls.get_data_path(persistence=persistence)
        path = os.path.join(data_path, subpath)
        delete_path(path)

    @staticmethod
    def get_outputs_path(persistence):
        import conf
        from stores.validators import validate_persistence_outputs

        persistence_outputs = validate_persistence_outputs(persistence_outputs=persistence)
        if persistence_outputs not in conf.get('PERSISTENCE_OUTPUTS'):
            raise VolumeNotFoundError('Outputs volume with name `{}` was defined in specification, '
                                      'but was not found'.format(persistence_outputs))
        persistence_type_condition = (
            'mountPath' not in conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs] and
            'bucket' not in conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs]
        )
        if persistence_type_condition:
            raise VolumeNotFoundError(
                'Outputs volume with name `{}` '
                'does not define a mountPath or bucket.'.format(persistence_outputs))

        return (conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs].get('mountPath') or
                conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs].get('bucket'))

    @classmethod
    def delete_outputs_path(cls, subpath, persistence):
        from stores.validators import validate_persistence_outputs

        persistence_outputs = validate_persistence_outputs(persistence_outputs=persistence)
        outputs_path = cls.get_outputs_path(persistence=persistence_outputs)
        path = os.path.join(outputs_path, subpath)
        store = cls.get_outputs_store(persistence_outputs=persistence_outputs)
        store.delete(path)

    @staticmethod
    def get_logs_path(persistence='default'):
        import conf

        persistence_type_condition = (
            'mountPath' not in conf.get('PERSISTENCE_LOGS') and
            'bucket' not in conf.get('PERSISTENCE_LOGS')
        )
        if persistence_type_condition:
            raise VolumeNotFoundError('Logs volume does not define a mountPath or bucket.')

        return (conf.get('PERSISTENCE_LOGS').get('bucket') or
                conf.get('PERSISTENCE_LOGS').get('mountPath'))

    @classmethod
    def delete_logs_path(cls, subpath, persistence='default'):
        logs_path = cls.get_logs_path(persistence=persistence)
        path = os.path.join(logs_path, subpath)
        store = cls.get_logs_store(persistence_logs=persistence)
        store.delete(path)

    @staticmethod
    def _get_store(store, secret_key):
        if not store or not secret_key:
            return StoreManager()
        try:
            store_access = config.get_dict(secret_key)
        except RheaError:
            raise VolumeNotFoundError(
                'Could not create store for path,'
                'received a store type `{}` without valid access key.'.format(store))

        return StoreManager.get_for_type(store_type=store, store_access=store_access)

    @classmethod
    def get_outputs_store(cls, persistence_outputs):
        import conf

        store, _, secret_key = get_store_secret_for_persistence(
            volume_name=persistence_outputs,
            volume_settings=conf.get('PERSISTENCE_OUTPUTS'))
        return cls._get_store(store, secret_key)

    @classmethod
    def get_logs_store(cls, persistence_logs='default'):
        import conf

        store, _, secret_key = get_store_secret_from_definition(conf.get('PERSISTENCE_LOGS'))
        return cls._get_store(store, secret_key)

    @classmethod
    def is_bucket_logs_persistence(cls, persistence='default'):
        import conf

        return bool(conf.get('PERSISTENCE_LOGS').get('bucket'))

    @classmethod
    def get_experiment_group_outputs_path(cls, experiment_group_name, persistence):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_group_name.split('.')
        values.insert(2, 'groups')
        return os.path.join(persistence_outputs, '/'.join(values))

    @classmethod
    def get_experiment_group_logs_path(cls, experiment_group_name, persistence='default'):
        values = experiment_group_name.split('.')
        values.insert(2, 'groups')
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_experiment_job_logs_path(cls, experiment_job_name, temp, persistence='default'):
        import conf

        values = experiment_job_name.split('.')
        values = values[:-2] + ['.'.join(values[-2:])]
        if len(values) == 4:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), '/'.join(values))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_experiment_outputs_path(cls,
                                    persistence,
                                    experiment_name,
                                    original_name=None,
                                    cloning_strategy=None):
        from db.models.cloning_strategies import CloningStrategy

        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_name.split('.')
        if original_name is not None and cloning_strategy == CloningStrategy.RESUME:
            values = original_name.split('.')
        if len(values) == 3:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')
        return os.path.join(persistence_outputs, '/'.join(values))

    @classmethod
    def get_experiment_logs_path(cls, experiment_name, temp, persistence='default'):
        import conf

        values = experiment_name.split('.')
        if len(values) == 3:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), '/'.join(values))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_job_outputs_path(cls, persistence, job_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        return os.path.join(persistence_outputs, job_name.replace('.', '/'))

    @classmethod
    def get_job_logs_path(cls, job_name, temp, persistence='default'):
        import conf

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), job_name.replace('.', '/'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, job_name.replace('.', '/'))

    @classmethod
    def upload_experiment_job_logs(cls, experiment_job_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_experiment_job_logs_path(experiment_job_name=experiment_job_name,
                                                     temp=True,
                                                     persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_experiment_job_logs_path(experiment_job_name=experiment_job_name,
                                                     temp=False,
                                                     persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_experiment_job_logs_path(cls, experiment_job_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_experiment_job_path(experiment_job_name, conf.get('LOGS_ARCHIVE_ROOT'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_experiment_job_path(experiment_job_name, persistence_logs)

    @classmethod
    def get_project_outputs_path(cls, persistence, project_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        return os.path.join(persistence_outputs, project_name.replace('.', '/'))

    @classmethod
    def get_project_logs_path(cls, project_name, persistence='default'):
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, project_name.replace('.', '/'))

    @classmethod
    def upload_experiment_logs(cls, experiment_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_experiment_logs_path(experiment_name=experiment_name,
                                                 temp=True,
                                                 persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_experiment_logs_path(experiment_name=experiment_name,
                                                 temp=False,
                                                 persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_experiment_logs_path(cls, experiment_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_experiment_path(experiment_name, conf.get('LOGS_ARCHIVE_ROOT'))

        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_experiment_path(experiment_name, persistence_logs)

    @classmethod
    def create_experiment_outputs_path(cls, persistence, experiment_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_name.split('.')
        path = create_experiment_path(experiment_name, persistence_outputs)
        path = os.path.join(path, values[-1])
        if not os.path.isdir(path):
            create_path(path)
        return path

    @classmethod
    def copy_experiment_outputs(cls,
                                persistence_outputs_from,
                                persistence_outputs_to,
                                experiment_name_from,
                                experiment_name_to):
        path_from = cls.get_experiment_outputs_path(persistence_outputs_from,
                                                    experiment_name_from)
        path_to = cls.get_experiment_outputs_path(persistence_outputs_to, experiment_name_to)
        shutil.copytree(path_from, path_to)

    @classmethod
    def upload_job_logs(cls, job_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_job_logs_path(job_name=job_name, temp=True, persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_job_logs_path(job_name=job_name, temp=False, persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_job_logs_path(cls, job_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_job_path(job_name, conf.get('LOGS_ARCHIVE_ROOT'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_job_path(job_name, persistence_logs)

    @classmethod
    def create_job_outputs_path(cls, persistence, job_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = job_name.split('.')
        path = create_job_path(job_name, persistence_outputs)
        path = os.path.join(path, values[-1])
        if not os.path.isdir(path):
            create_path(path)
        return path

    @staticmethod
    def _validate_persistence(persistence, persistence_name, persistence_type):
        try:
            VolumeConfig.from_dict(persistence)
        except ValidationError:
            try:
                StoreConfig.from_dict(persistence)
            except (ValidationError, TypeError):
                raise InvalidService('Persistence `{}`, of type `{}`, is not valid.'.format(
                    persistence_name, persistence_type
                ))

    def _validate_logs(self):
        import conf

        self._validate_persistence(persistence=conf.get('PERSISTENCE_LOGS'),
                                   persistence_name='default',
                                   persistence_type='PERSISTENCE_LOGS')

    def _validate_outputs(self):
        import conf

        for persistence_name, persistence in conf.get('PERSISTENCE_OUTPUTS').items():
            self._validate_persistence(persistence=persistence,
                                       persistence_name=persistence_name,
                                       persistence_type='PERSISTENCE_OUTPUTS')

    def _validate_data(self):
        import conf

        for persistence_name, persistence in conf.get('PERSISTENCE_DATA').items():
            self._validate_persistence(persistence=persistence,
                                       persistence_name=persistence_name,
                                       persistence_type='PERSISTENCE_DATA')

    def validate(self):
        self._validate_logs()
        self._validate_outputs()
        self._validate_data()
<commit_msg>Update stores service to handle non existing buckets on delete<commit_after>import os
import shutil

from hestia.paths import check_or_create_path, create_path, delete_path
from hestia.service_interface import InvalidService, Service
from marshmallow import ValidationError
from polystores import StoreManager
from polystores.exceptions import PolyaxonStoresException
from rhea import RheaError

from libs.paths.experiment_jobs import create_experiment_job_path
from libs.paths.experiments import create_experiment_path
from libs.paths.jobs import create_job_path
from polyaxon.config_manager import config
from stores.exceptions import VolumeNotFoundError
from stores.schemas.store import StoreConfig
from stores.schemas.volume import VolumeConfig
from stores.store_secrets import get_store_secret_for_persistence, get_store_secret_from_definition

class StoresService(Service):
    __all__ = (
        'get_data_paths',
        'get_data_path',
        'delete_data_path',
        'get_outputs_path',
        'delete_outputs_path',
        'get_logs_path',
        'delete_logs_path',
        'get_outputs_store',
        'get_logs_store',
        'is_bucket_logs_persistence',
        'get_experiment_group_outputs_path',
        'get_experiment_group_logs_path',
        'get_experiment_job_logs_path',
        'upload_experiment_job_logs',
        'get_experiment_outputs_path',
        'get_experiment_logs_path',
        'get_job_outputs_path',
        'get_job_logs_path',
        'upload_job_logs',
        'get_project_outputs_path',
        'get_project_logs_path',
        'create_experiment_logs_path',
        'create_experiment_outputs_path',
        'create_experiment_job_logs_path',
        'upload_experiment_logs',
        'copy_experiment_outputs',
        'create_job_logs_path',
        'create_job_outputs_path'
    )

    @staticmethod
    def get_data_paths(persistences):
        import conf
        from stores.validators import validate_persistence_data

        persistence_data = validate_persistence_data(persistence_data=persistences)
        persistence_paths = {}
        for persistence in persistence_data:
            if persistence not in conf.get('PERSISTENCE_DATA'):
                raise VolumeNotFoundError(
                    'Data volume with name `{}` was defined in specification, '
                    'but was not found'.format(persistence))
            persistence_type_condition = (
                'mountPath' not in conf.get('PERSISTENCE_DATA')[persistence] and
                'bucket' not in conf.get('PERSISTENCE_DATA')[persistence]
            )
            if persistence_type_condition:
                raise VolumeNotFoundError(
                    'Data volume with name `{}` '
                    'does not define a mountPath or bucket.'.format(persistence))

            persistence_paths[persistence] = (
                conf.get('PERSISTENCE_DATA')[persistence].get('mountPath') or
                conf.get('PERSISTENCE_DATA')[persistence].get('bucket'))

        return persistence_paths

    @classmethod
    def get_data_path(cls, persistence):
        persistences = [persistence] if persistence else None
        paths = cls.get_data_paths(persistences=persistences)
        return list(paths.values())[0]

    @classmethod
    def delete_data_path(cls, subpath, persistence):
        data_path = cls.get_data_path(persistence=persistence)
        path = os.path.join(data_path, subpath)
        delete_path(path)

    @staticmethod
    def get_outputs_path(persistence):
        import conf
        from stores.validators import validate_persistence_outputs

        persistence_outputs = validate_persistence_outputs(persistence_outputs=persistence)
        if persistence_outputs not in conf.get('PERSISTENCE_OUTPUTS'):
            raise VolumeNotFoundError('Outputs volume with name `{}` was defined in specification, '
                                      'but was not found'.format(persistence_outputs))
        persistence_type_condition = (
            'mountPath' not in conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs] and
            'bucket' not in conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs]
        )
        if persistence_type_condition:
            raise VolumeNotFoundError(
                'Outputs volume with name `{}` '
                'does not define a mountPath or bucket.'.format(persistence_outputs))

        return (conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs].get('mountPath') or
                conf.get('PERSISTENCE_OUTPUTS')[persistence_outputs].get('bucket'))

    @classmethod
    def delete_outputs_path(cls, subpath, persistence):
        from stores.validators import validate_persistence_outputs

        persistence_outputs = validate_persistence_outputs(persistence_outputs=persistence)
        outputs_path = cls.get_outputs_path(persistence=persistence_outputs)
        path = os.path.join(outputs_path, subpath)
        store = cls.get_outputs_store(persistence_outputs=persistence_outputs)
        try:
            store.delete(path)
        except (PolyaxonStoresException, VolumeNotFoundError):
            pass

    @staticmethod
    def get_logs_path(persistence='default'):
        import conf

        persistence_type_condition = (
            'mountPath' not in conf.get('PERSISTENCE_LOGS') and
            'bucket' not in conf.get('PERSISTENCE_LOGS')
        )
        if persistence_type_condition:
            raise VolumeNotFoundError('Logs volume does not define a mountPath or bucket.')

        return (conf.get('PERSISTENCE_LOGS').get('bucket') or
                conf.get('PERSISTENCE_LOGS').get('mountPath'))

    @classmethod
    def delete_logs_path(cls, subpath, persistence='default'):
        logs_path = cls.get_logs_path(persistence=persistence)
        path = os.path.join(logs_path, subpath)
        store = cls.get_logs_store(persistence_logs=persistence)
        try:
            store.delete(path)
        except (PolyaxonStoresException, VolumeNotFoundError):
            pass

    @staticmethod
    def _get_store(store, secret_key):
        if not store or not secret_key:
            return StoreManager()
        try:
            store_access = config.get_dict(secret_key)
        except RheaError:
            raise VolumeNotFoundError(
                'Could not create store for path,'
                'received a store type `{}` without valid access key.'.format(store))

        return StoreManager.get_for_type(store_type=store, store_access=store_access)

    @classmethod
    def get_outputs_store(cls, persistence_outputs):
        import conf

        store, _, secret_key = get_store_secret_for_persistence(
            volume_name=persistence_outputs,
            volume_settings=conf.get('PERSISTENCE_OUTPUTS'))
        return cls._get_store(store, secret_key)

    @classmethod
    def get_logs_store(cls, persistence_logs='default'):
        import conf

        store, _, secret_key = get_store_secret_from_definition(conf.get('PERSISTENCE_LOGS'))
        return cls._get_store(store, secret_key)

    @classmethod
    def is_bucket_logs_persistence(cls, persistence='default'):
        import conf

        return bool(conf.get('PERSISTENCE_LOGS').get('bucket'))

    @classmethod
    def get_experiment_group_outputs_path(cls, experiment_group_name, persistence):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_group_name.split('.')
        values.insert(2, 'groups')
        return os.path.join(persistence_outputs, '/'.join(values))

    @classmethod
    def get_experiment_group_logs_path(cls, experiment_group_name, persistence='default'):
        values = experiment_group_name.split('.')
        values.insert(2, 'groups')
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_experiment_job_logs_path(cls, experiment_job_name, temp, persistence='default'):
        import conf

        values = experiment_job_name.split('.')
        values = values[:-2] + ['.'.join(values[-2:])]
        if len(values) == 4:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), '/'.join(values))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_experiment_outputs_path(cls,
                                    persistence,
                                    experiment_name,
                                    original_name=None,
                                    cloning_strategy=None):
        from db.models.cloning_strategies import CloningStrategy

        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_name.split('.')
        if original_name is not None and cloning_strategy == CloningStrategy.RESUME:
            values = original_name.split('.')
        if len(values) == 3:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')
        return os.path.join(persistence_outputs, '/'.join(values))

    @classmethod
    def get_experiment_logs_path(cls, experiment_name, temp, persistence='default'):
        import conf

        values = experiment_name.split('.')
        if len(values) == 3:
            values.insert(2, 'experiments')
        else:
            values.insert(2, 'groups')

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), '/'.join(values))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, '/'.join(values))

    @classmethod
    def get_job_outputs_path(cls, persistence, job_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        return os.path.join(persistence_outputs, job_name.replace('.', '/'))

    @classmethod
    def get_job_logs_path(cls, job_name, temp, persistence='default'):
        import conf

        if temp:
            return os.path.join(conf.get('LOGS_ARCHIVE_ROOT'), job_name.replace('.', '/'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, job_name.replace('.', '/'))

    @classmethod
    def upload_experiment_job_logs(cls, experiment_job_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_experiment_job_logs_path(experiment_job_name=experiment_job_name,
                                                     temp=True,
                                                     persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_experiment_job_logs_path(experiment_job_name=experiment_job_name,
                                                     temp=False,
                                                     persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_experiment_job_logs_path(cls, experiment_job_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_experiment_job_path(experiment_job_name, conf.get('LOGS_ARCHIVE_ROOT'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_experiment_job_path(experiment_job_name, persistence_logs)

    @classmethod
    def get_project_outputs_path(cls, persistence, project_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        return os.path.join(persistence_outputs, project_name.replace('.', '/'))

    @classmethod
    def get_project_logs_path(cls, project_name, persistence='default'):
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return os.path.join(persistence_logs, project_name.replace('.', '/'))

    @classmethod
    def upload_experiment_logs(cls, experiment_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_experiment_logs_path(experiment_name=experiment_name,
                                                 temp=True,
                                                 persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_experiment_logs_path(experiment_name=experiment_name,
                                                 temp=False,
                                                 persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_experiment_logs_path(cls, experiment_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_experiment_path(experiment_name, conf.get('LOGS_ARCHIVE_ROOT'))

        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_experiment_path(experiment_name, persistence_logs)

    @classmethod
    def create_experiment_outputs_path(cls, persistence, experiment_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = experiment_name.split('.')
        path = create_experiment_path(experiment_name, persistence_outputs)
        path = os.path.join(path, values[-1])
        if not os.path.isdir(path):
            create_path(path)
        return path

    @classmethod
    def copy_experiment_outputs(cls,
                                persistence_outputs_from,
                                persistence_outputs_to,
                                experiment_name_from,
                                experiment_name_to):
        path_from = cls.get_experiment_outputs_path(persistence_outputs_from,
                                                    experiment_name_from)
        path_to = cls.get_experiment_outputs_path(persistence_outputs_to, experiment_name_to)
        shutil.copytree(path_from, path_to)

    @classmethod
    def upload_job_logs(cls, job_name, persistence='default'):
        store = cls.get_logs_store(persistence_logs=persistence)
        temp_path = cls.get_job_logs_path(job_name=job_name, temp=True, persistence=persistence)
        if not os.path.exists(temp_path):
            return
        logs_path = cls.get_job_logs_path(job_name=job_name, temp=False, persistence=persistence)
        store.upload_file(filename=temp_path, path=logs_path, use_basename=False)

    @classmethod
    def create_job_logs_path(cls, job_name, temp, persistence='default'):
        import conf

        if temp:
            check_or_create_path(conf.get('LOGS_ARCHIVE_ROOT'))
            return create_job_path(job_name, conf.get('LOGS_ARCHIVE_ROOT'))
        persistence_logs = cls.get_logs_path(persistence=persistence)
        return create_job_path(job_name, persistence_logs)

    @classmethod
    def create_job_outputs_path(cls, persistence, job_name):
        persistence_outputs = cls.get_outputs_path(persistence=persistence)
        values = job_name.split('.')
        path = create_job_path(job_name, persistence_outputs)
        path = os.path.join(path, values[-1])
        if not os.path.isdir(path):
            create_path(path)
        return path

    @staticmethod
    def _validate_persistence(persistence, persistence_name, persistence_type):
        try:
            VolumeConfig.from_dict(persistence)
        except ValidationError:
            try:
                StoreConfig.from_dict(persistence)
            except (ValidationError, TypeError):
                raise InvalidService('Persistence `{}`, of type `{}`, is not valid.'.format(
                    persistence_name, persistence_type
                ))

    def _validate_logs(self):
        import conf

        self._validate_persistence(persistence=conf.get('PERSISTENCE_LOGS'),
                                   persistence_name='default',
                                   persistence_type='PERSISTENCE_LOGS')

    def _validate_outputs(self):
        import conf

        for persistence_name, persistence in conf.get('PERSISTENCE_OUTPUTS').items():
            self._validate_persistence(persistence=persistence,
                                       persistence_name=persistence_name,
                                       persistence_type='PERSISTENCE_OUTPUTS')

    def _validate_data(self):
        import conf

        for persistence_name, persistence in conf.get('PERSISTENCE_DATA').items():
            self._validate_persistence(persistence=persistence,
                                       persistence_name=persistence_name,
                                       persistence_type='PERSISTENCE_DATA')

    def validate(self):
        self._validate_logs()
        self._validate_outputs()
        self._validate_data()
<|endoftext|>


<commit_before>import sys, unittest, re, os.path, warnings
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'src'))

warnings.simplefilter('ignore', DeprecationWarning)

import shutil
import time
import ctypes
from functools import partial
from tempfile import mkdtemp
from multiprocessing import Value
from multiprocessing.managers import BaseManager
from Exscript import Queue, Account, AccountPool, FileLogger
from Exscript.Connection import Connection
from Exscript.protocols import Dummy
from Exscript.interpreter.Exception import FailException
from Exscript.util.decorator import bind
from Exscript.util.log import log_to

def count_calls(job, data, **kwargs):
    assert hasattr(job, 'start')
    assert kwargs.has_key('testarg')
    data.value += 1

def count_calls2(job, conn, data, **kwargs):
    assert isinstance(conn, Connection)
    count_calls(job, data, **kwargs)

def spawn_subtask(job, conn, queue, data, **kwargs):
    count_calls2(job, conn, data, **kwargs)
    func  = bind(count_calls2, data, testarg = 1)
    task  = queue.priority_run('subtask', func)
    task.wait()

def do_nothing(job, conn):
    pass

def say_hello(job, conn):
    conn.send('hello')

def error(job, conn):
    say_hello(job, conn)
    raise FailException('intentional error')

def fatal_error(job, conn):
    say_hello(job, conn)
    raise Exception('intentional fatal error')

class MyProtocol(Dummy):
    pass

def raise_if_not_myprotocol(job, conn):
    if not isinstance(conn, MyProtocol):
        raise Exception('not a MyProtocol instance')

class Log(object):
    data = ''

    def write(self, data):
        self.data += data

    def flush(self):
        pass

    def read(self):
        return self.data

class LogManager(BaseManager):
    pass
LogManager.register('Log', Log)

class QueueTest(unittest.TestCase):
    CORRELATE = Queue
    mode = 'threading'

    def createQueue(self, logdir = None, **kwargs):
        if self.queue:
            self.queue.destroy()
        self.out   = self.manager.Log()
        self.err   = self.manager.Log()
        self.queue = Queue(mode   = self.mode,
                           stdout = self.out,
                           stderr = self.err,
                           **kwargs)
        self.accm  = self.queue.account_manager
        if logdir is not None:
            self.logger = FileLogger(logdir)

    def setUp(self):
        self.tempdir = mkdtemp()
        self.queue   = None
        self.logger  = None
        self.manager = LogManager()
        self.manager.start()
        self.createQueue(verbose = -1, logdir = self.tempdir)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        try:
            self.queue.destroy()
        except:
            pass # queue already destroyed
        self.manager.shutdown()

    def assertVerbosity(self, channel, expected):
        data = channel.read()
        if expected == 'no_tb':
            self.assert_('error' in data, data)
            self.assert_('Traceback' not in data)
        elif expected == 'tb':
            self.assert_('error' in data, data)
            self.assert_('Traceback' in data)
        elif expected == '':
            self.assertEqual(data, '')
        else:
            msg = repr(expected) + ' not in ' + repr(data)
            self.assert_(expected in data, msg)

    def testConstructor(self):
        self.assertEqual(1, self.queue.get_max_threads())

        @log_to(self.logger)
        def logged_hello(job, conn):
            say_hello(job, conn)

        # Test all verbosity levels.
        levels = (
            (-1, 1, ('',      ''), ('',      ''),      ('',      'tb')),
            (-1, 2, ('',      ''), ('',      ''),      ('',      'tb')),
            (0,  1, ('',      ''), ('',      'no_tb'), ('',      'tb')),
            (0,  2, ('',      ''), ('',      'no_tb'), ('',      'tb')),
            (1,  1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
            (1,  2, ('[',     ''), ('[',     'no_tb'), ('[',     'tb')),
            (2,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (2,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (3,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (3,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (4,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (4,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (5,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (5,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
        )
        for level, max_threads, with_simple, with_error, with_fatal in levels:
            stdout, stderr = with_simple
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', logged_hello)
            self.queue.join()
            #print level, max_threads, with_simple, with_error, with_fatal
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

            stdout, stderr = with_error
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', error)
            self.queue.join()
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

            stdout, stderr = with_fatal
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', fatal_error)
            self.queue.join()
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

    def testCreatePipe(self):
        account = Account('user', 'test')
        self.accm.add_account(account)
        pipe = self.queue._create_pipe()
        pipe.send(('acquire-account', None))
        response = pipe.recv()
        expected = (account.__hash__(),
                    account.get_name(),
                    account.get_password(),
                    account.get_authorization_password(),
                    account.get_key())
        self.assertEqual(response, expected)
        pipe.send(('release-account', account.__hash__()))
        response = pipe.recv()
        self.assertEqual(response, 'ok')

    def testSetMaxThreads(self):
        self.assertEqual(1, self.queue.get_max_threads())
        self.queue.set_max_threads(2)
        self.assertEqual(2, self.queue.get_max_threads())

    def testGetMaxThreads(self):
        pass # Already tested in testSetMaxThreads().

    def testGetProgress(self):
        self.assertEqual(0.0, self.queue.get_progress())
        self.testIsCompleted()
        self.assertEqual(100.0, self.queue.get_progress())

    def testAddAccount(self):
        self.assertEqual(0, self.accm.default_pool.n_accounts())
        account = Account('user', 'test')
        self.queue.add_account(account)
        self.assertEqual(1, self.accm.default_pool.n_accounts())

    def testAddAccountPool(self):
        self.assertEqual(0, self.accm.default_pool.n_accounts())
        account = Account('user', 'test')
        self.queue.add_account(account)
        self.assertEqual(1, self.accm.default_pool.n_accounts())

        def match_cb(data, host):
            data['match-called'].value = True
            return True

        def start_cb(data, job, conn):
            account = conn.acquire_account()
            data['start-called'].value = True
            data['account-hash'].value = account.__hash__()
            account.release()

        # Replace the default pool.
        pool1 = AccountPool()
        self.queue.add_account_pool(pool1)
        self.assertEqual(self.accm.default_pool, pool1)

        # Add another pool, making sure that it does not replace
        # the default pool.
        pool2    = AccountPool()
        account2 = Account('user', 'test')
        pool2.add_account(account2)

        match_called = Value(ctypes.c_bool, False)
        start_called = Value(ctypes.c_bool, False)
        account_hash = Value(ctypes.c_long, account2.__hash__())
        data = {'match-called': match_called,
                'start-called': start_called,
                'account-hash': account_hash}
        self.queue.add_account_pool(pool2, partial(match_cb, data))
        self.assertEqual(self.accm.default_pool, pool1)

        # Make sure that pool2 is chosen (because the match function
        # returns True).
        self.queue.run('dummy', partial(start_cb, data))
        self.queue.shutdown()
        data = dict((k, v.value) for (k, v) in data.iteritems())
        self.assertEqual(data, {'match-called': True,
                                'start-called': True,
                                'account-hash': account2.__hash__()})

    def startTask(self):
        self.testAddAccount()
        hosts = ['dummy1', 'dummy2']
        task  = self.queue.run(hosts, log_to(self.logger)(do_nothing))
        self.assert_(task is not None)
        return task

    def testIsCompleted(self):
        self.assert_(self.queue.is_completed())
        task = self.startTask()
        self.failIf(self.queue.is_completed())
        task.wait()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())

    def testJoin(self):
        task = self.startTask()
        self.queue.join()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())

    def testShutdown(self):
        task = self.startTask()   # this also adds an account
        self.queue.shutdown()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())
        self.assertEqual(self.accm.default_pool.n_accounts(), 1)

    def testDestroy(self):
        task = self.startTask()   # this also adds an account
        self.queue.destroy()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())
        self.assertEqual(self.accm.default_pool.n_accounts(), 0)

    def testReset(self):
        self.testAddAccount()
        self.queue.reset()
        self.assertEqual(self.accm.default_pool.n_accounts(), 0)

    def testRun(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.run(hosts,    func)
        self.queue.run('dummy3', func)
        self.queue.shutdown()
        self.assertEqual(data.value, 3)

        self.queue.run('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 4)

    def testRunOrIgnore(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2', 'dummy1']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.workqueue.pause()
        self.queue.run_or_ignore(hosts,    func)
        self.queue.run_or_ignore('dummy2', func)
        self.queue.workqueue.unpause()
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.run_or_ignore('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    def testPriorityRun(self):
        def write(data, value, *args):
            data.value = value

        data = Value('i', 0)
        self.queue.workqueue.pause()
        self.queue.enqueue(partial(write, data, 1))
        self.queue.priority_run('dummy', partial(write, data, 2))
        self.queue.workqueue.unpause()
        self.queue.destroy()

        # The 'dummy' job should run first, so the value must
        # be overwritten by the other process.
        self.assertEqual(data.value, 1)

    def testPriorityRunOrRaise(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2', 'dummy1']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.workqueue.pause()
        self.queue.priority_run_or_raise(hosts,    func)
        self.queue.priority_run_or_raise('dummy2', func)
        self.queue.workqueue.unpause()
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.priority_run_or_raise('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    def testForceRun(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2']
        func  = bind(count_calls2, data, testarg = 1)

        # By setting max_threads to 0 we ensure that the 'force' part is
        # actually tested; the thread should run regardless.
        self.queue.set_max_threads(0)
        self.queue.force_run(hosts, func)
        self.queue.destroy()
        self.assertEqual(data.value, 2)

    def testEnqueue(self):
        from functools import partial
        data = Value('i', 0)
        func = bind(count_calls, data, testarg = 1)
        self.queue.enqueue(func)
        self.queue.enqueue(func)
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.enqueue(func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    #FIXME: Not a method test; this should probably be elsewhere.
    def testLogging(self):
        task = self.startTask()
        while not task.is_completed():
            time.sleep(.1)

        # The following function is not synchronous with the above, so add
        # a timeout to avoid races.
        time.sleep(.1)
        self.assert_(self.queue.is_completed())

        logfiles = os.listdir(self.tempdir)
        self.assertEqual(2, len(logfiles))
        self.assert_('dummy1.log' in logfiles)
        self.assert_('dummy2.log' in logfiles)
        for file in logfiles:
            content = open(os.path.join(self.tempdir, file)).read()

class QueueTestMultiProcessing(QueueTest):
    mode = 'multiprocessing'

def suite():
    loader = unittest.TestLoader()
    suite1 = loader.loadTestsFromTestCase(QueueTest)
    suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing)
    return unittest.TestSuite((suite1, suite2))
if __name__ == '__main__':
    unittest.TextTestRunner(verbosity = 2).run(suite())
<commit_msg>remove some useless logging from QueueTest.<commit_after>import sys, unittest, re, os.path, warnings
sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', '..', 'src'))

warnings.simplefilter('ignore', DeprecationWarning)

import shutil
import time
import ctypes
from functools import partial
from tempfile import mkdtemp
from multiprocessing import Value
from multiprocessing.managers import BaseManager
from Exscript import Queue, Account, AccountPool, FileLogger
from Exscript.Connection import Connection
from Exscript.protocols import Dummy
from Exscript.interpreter.Exception import FailException
from Exscript.util.decorator import bind
from Exscript.util.log import log_to

def count_calls(job, data, **kwargs):
    assert hasattr(job, 'start')
    assert kwargs.has_key('testarg')
    data.value += 1

def count_calls2(job, conn, data, **kwargs):
    assert isinstance(conn, Connection)
    count_calls(job, data, **kwargs)

def spawn_subtask(job, conn, queue, data, **kwargs):
    count_calls2(job, conn, data, **kwargs)
    func  = bind(count_calls2, data, testarg = 1)
    task  = queue.priority_run('subtask', func)
    task.wait()

def do_nothing(job, conn):
    pass

def say_hello(job, conn):
    conn.send('hello')

def error(job, conn):
    say_hello(job, conn)
    raise FailException('intentional error')

def fatal_error(job, conn):
    say_hello(job, conn)
    raise Exception('intentional fatal error')

class MyProtocol(Dummy):
    pass

def raise_if_not_myprotocol(job, conn):
    if not isinstance(conn, MyProtocol):
        raise Exception('not a MyProtocol instance')

class Log(object):
    data = ''

    def write(self, data):
        self.data += data

    def flush(self):
        pass

    def read(self):
        return self.data

class LogManager(BaseManager):
    pass
LogManager.register('Log', Log)

class QueueTest(unittest.TestCase):
    CORRELATE = Queue
    mode = 'threading'

    def createQueue(self, logdir = None, **kwargs):
        if self.queue:
            self.queue.destroy()
        self.out   = self.manager.Log()
        self.err   = self.manager.Log()
        self.queue = Queue(mode   = self.mode,
                           stdout = self.out,
                           stderr = self.err,
                           **kwargs)
        self.accm  = self.queue.account_manager
        if logdir is not None:
            self.logger = FileLogger(logdir)

    def setUp(self):
        self.tempdir = mkdtemp()
        self.queue   = None
        self.logger  = None
        self.manager = LogManager()
        self.manager.start()
        self.createQueue(verbose = -1, logdir = self.tempdir)

    def tearDown(self):
        shutil.rmtree(self.tempdir)
        try:
            self.queue.destroy()
        except:
            pass # queue already destroyed
        self.manager.shutdown()

    def assertVerbosity(self, channel, expected):
        data = channel.read()
        if expected == 'no_tb':
            self.assert_('error' in data, data)
            self.assert_('Traceback' not in data)
        elif expected == 'tb':
            self.assert_('error' in data, data)
            self.assert_('Traceback' in data)
        elif expected == '':
            self.assertEqual(data, '')
        else:
            msg = repr(expected) + ' not in ' + repr(data)
            self.assert_(expected in data, msg)

    def testConstructor(self):
        self.assertEqual(1, self.queue.get_max_threads())

        # Test all verbosity levels.
        levels = (
            (-1, 1, ('',      ''), ('',      ''),      ('',      'tb')),
            (-1, 2, ('',      ''), ('',      ''),      ('',      'tb')),
            (0,  1, ('',      ''), ('',      'no_tb'), ('',      'tb')),
            (0,  2, ('',      ''), ('',      'no_tb'), ('',      'tb')),
            (1,  1, ('hello', ''), ('hello', 'no_tb'), ('hello', 'tb')),
            (1,  2, ('[',     ''), ('[',     'no_tb'), ('[',     'tb')),
            (2,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (2,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (3,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (3,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (4,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (4,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
            (5,  1, ('hello', ''), ('hello', 'tb'),    ('hello', 'tb')),
            (5,  2, ('[',     ''), ('[',     'tb'),    ('[',     'tb')),
        )
        for level, max_threads, with_simple, with_error, with_fatal in levels:
            stdout, stderr = with_simple
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', say_hello)
            self.queue.join()
            #print level, max_threads, with_simple, with_error, with_fatal
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

            stdout, stderr = with_error
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', error)
            self.queue.join()
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

            stdout, stderr = with_fatal
            self.createQueue(verbose = level, max_threads = max_threads)
            self.queue.run('dummy://mytest', fatal_error)
            self.queue.join()
            self.assertVerbosity(self.out, stdout)
            self.assertVerbosity(self.err, stderr)

    def testCreatePipe(self):
        account = Account('user', 'test')
        self.accm.add_account(account)
        pipe = self.queue._create_pipe()
        pipe.send(('acquire-account', None))
        response = pipe.recv()
        expected = (account.__hash__(),
                    account.get_name(),
                    account.get_password(),
                    account.get_authorization_password(),
                    account.get_key())
        self.assertEqual(response, expected)
        pipe.send(('release-account', account.__hash__()))
        response = pipe.recv()
        self.assertEqual(response, 'ok')

    def testSetMaxThreads(self):
        self.assertEqual(1, self.queue.get_max_threads())
        self.queue.set_max_threads(2)
        self.assertEqual(2, self.queue.get_max_threads())

    def testGetMaxThreads(self):
        pass # Already tested in testSetMaxThreads().

    def testGetProgress(self):
        self.assertEqual(0.0, self.queue.get_progress())
        self.testIsCompleted()
        self.assertEqual(100.0, self.queue.get_progress())

    def testAddAccount(self):
        self.assertEqual(0, self.accm.default_pool.n_accounts())
        account = Account('user', 'test')
        self.queue.add_account(account)
        self.assertEqual(1, self.accm.default_pool.n_accounts())

    def testAddAccountPool(self):
        self.assertEqual(0, self.accm.default_pool.n_accounts())
        account = Account('user', 'test')
        self.queue.add_account(account)
        self.assertEqual(1, self.accm.default_pool.n_accounts())

        def match_cb(data, host):
            data['match-called'].value = True
            return True

        def start_cb(data, job, conn):
            account = conn.acquire_account()
            data['start-called'].value = True
            data['account-hash'].value = account.__hash__()
            account.release()

        # Replace the default pool.
        pool1 = AccountPool()
        self.queue.add_account_pool(pool1)
        self.assertEqual(self.accm.default_pool, pool1)

        # Add another pool, making sure that it does not replace
        # the default pool.
        pool2    = AccountPool()
        account2 = Account('user', 'test')
        pool2.add_account(account2)

        match_called = Value(ctypes.c_bool, False)
        start_called = Value(ctypes.c_bool, False)
        account_hash = Value(ctypes.c_long, account2.__hash__())
        data = {'match-called': match_called,
                'start-called': start_called,
                'account-hash': account_hash}
        self.queue.add_account_pool(pool2, partial(match_cb, data))
        self.assertEqual(self.accm.default_pool, pool1)

        # Make sure that pool2 is chosen (because the match function
        # returns True).
        self.queue.run('dummy', partial(start_cb, data))
        self.queue.shutdown()
        data = dict((k, v.value) for (k, v) in data.iteritems())
        self.assertEqual(data, {'match-called': True,
                                'start-called': True,
                                'account-hash': account2.__hash__()})

    def startTask(self):
        self.testAddAccount()
        hosts = ['dummy1', 'dummy2']
        task  = self.queue.run(hosts, log_to(self.logger)(do_nothing))
        self.assert_(task is not None)
        return task

    def testIsCompleted(self):
        self.assert_(self.queue.is_completed())
        task = self.startTask()
        self.failIf(self.queue.is_completed())
        task.wait()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())

    def testJoin(self):
        task = self.startTask()
        self.queue.join()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())

    def testShutdown(self):
        task = self.startTask()   # this also adds an account
        self.queue.shutdown()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())
        self.assertEqual(self.accm.default_pool.n_accounts(), 1)

    def testDestroy(self):
        task = self.startTask()   # this also adds an account
        self.queue.destroy()
        self.assert_(task.is_completed())
        self.assert_(self.queue.is_completed())
        self.assertEqual(self.accm.default_pool.n_accounts(), 0)

    def testReset(self):
        self.testAddAccount()
        self.queue.reset()
        self.assertEqual(self.accm.default_pool.n_accounts(), 0)

    def testRun(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.run(hosts,    func)
        self.queue.run('dummy3', func)
        self.queue.shutdown()
        self.assertEqual(data.value, 3)

        self.queue.run('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 4)

    def testRunOrIgnore(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2', 'dummy1']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.workqueue.pause()
        self.queue.run_or_ignore(hosts,    func)
        self.queue.run_or_ignore('dummy2', func)
        self.queue.workqueue.unpause()
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.run_or_ignore('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    def testPriorityRun(self):
        def write(data, value, *args):
            data.value = value

        data = Value('i', 0)
        self.queue.workqueue.pause()
        self.queue.enqueue(partial(write, data, 1))
        self.queue.priority_run('dummy', partial(write, data, 2))
        self.queue.workqueue.unpause()
        self.queue.destroy()

        # The 'dummy' job should run first, so the value must
        # be overwritten by the other process.
        self.assertEqual(data.value, 1)

    def testPriorityRunOrRaise(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2', 'dummy1']
        func  = bind(count_calls2, data, testarg = 1)
        self.queue.workqueue.pause()
        self.queue.priority_run_or_raise(hosts,    func)
        self.queue.priority_run_or_raise('dummy2', func)
        self.queue.workqueue.unpause()
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.priority_run_or_raise('dummy4', func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    def testForceRun(self):
        data  = Value('i', 0)
        hosts = ['dummy1', 'dummy2']
        func  = bind(count_calls2, data, testarg = 1)

        # By setting max_threads to 0 we ensure that the 'force' part is
        # actually tested; the thread should run regardless.
        self.queue.set_max_threads(0)
        self.queue.force_run(hosts, func)
        self.queue.destroy()
        self.assertEqual(data.value, 2)

    def testEnqueue(self):
        from functools import partial
        data = Value('i', 0)
        func = bind(count_calls, data, testarg = 1)
        self.queue.enqueue(func)
        self.queue.enqueue(func)
        self.queue.shutdown()
        self.assertEqual(data.value, 2)

        self.queue.enqueue(func)
        self.queue.destroy()
        self.assertEqual(data.value, 3)

    #FIXME: Not a method test; this should probably be elsewhere.
    def testLogging(self):
        task = self.startTask()
        while not task.is_completed():
            time.sleep(.1)

        # The following function is not synchronous with the above, so add
        # a timeout to avoid races.
        time.sleep(.1)
        self.assert_(self.queue.is_completed())

        logfiles = os.listdir(self.tempdir)
        self.assertEqual(2, len(logfiles))
        self.assert_('dummy1.log' in logfiles)
        self.assert_('dummy2.log' in logfiles)
        for file in logfiles:
            content = open(os.path.join(self.tempdir, file)).read()

class QueueTestMultiProcessing(QueueTest):
    mode = 'multiprocessing'

def suite():
    loader = unittest.TestLoader()
    suite1 = loader.loadTestsFromTestCase(QueueTest)
    suite2 = loader.loadTestsFromTestCase(QueueTestMultiProcessing)
    return unittest.TestSuite((suite1, suite2))
if __name__ == '__main__':
    unittest.TextTestRunner(verbosity = 2).run(suite())
<|endoftext|>


<commit_before># NetworKit native classes and functions
from _NetworKit import GraphProperties, ConnectedComponents, ParallelConnectedComponents, StronglyConnectedComponents, ClusteringCoefficient, Diameter, Eccentricity, CoreDecomposition

# other submodules
from . import community
from . import termgraph
from . import auxiliary
from . import nxadapter
from . import powerlaw

# other modules
import textwrap
import collections
import math
import logging

try:
	import networkx as nx
except ImportError:
	logging.warning("""WARNING: module 'networkx' not installed, which is required by some
						functions.""")
try:
	import tabulate
except ImportError:
	logging.warning("""WARNING: module 'tabulate' not installed, which is required by some
						functions. In order to install 'tabulate', Python 3.3 is required""")

try:
	from scipy import stats
except ImportError:
	logging.warning("""WARNING: module 'scipy' not installed, which is required by some
						functions.""")

########  PROPERTIES ########

def size(G):
	""" Return number of nodes and number of edges"""
	n = G.numberOfNodes()
	m = G.numberOfEdges()
	return (n, m)

def degrees(G):
	""" Return min/max/avg degree"""
	minMaxDeg = GraphProperties.minMaxDegree(G)
	avgDeg = GraphProperties.averageDegree(G)
	return (minMaxDeg[0], minMaxDeg[1], avgDeg)

def degreeDistribution(G):
	""" Return the degree distribution of a graph"""
	return GraphProperties.degreeDistribution(G)

def degreeSequence(G):
	""" Return the degree sequence of a graph"""
	return GraphProperties.degreeSequence(G)

def density(G):
	""" Return the density of the graph"""
	(n, m) = size(G)
	return (2 * m) / (n * (n-1))

def components(G):
	""" Find and analyze detected components.
		Returns the number of components and the sizes
		of each component. For more details use the
		ConnectedComponents class.
	"""
	logging.info("[...] finding connected components....")
	if G.isDirected():
		cc = StronglyConnectedComponents(G)
	else:
		cc = ConnectedComponents(G)
	cc.run()
	components = cc.getPartition()
	nComponents = components.numberOfSubsets()
	componentSizes = components.subsetSizeMap()
	return (nComponents, componentSizes)

def numberOfComponents(G):
	""" Find and number of components """
	logging.info("[...] finding connected components....")
	cc = ConnectedComponents()
	cc.run(G)
	nComponents = cc.numberOfComponents()
	return nComponents

def clustering(G, error=0.01):
	"""
		Returns approximate average local clustering coefficient
		The maximum error can be given as a parameter and determines
		the number of samples taken.

		for details see:
			Schank, Wagner: Approximating Clustering Coefficient and Transitivity
	"""
	nSamples = math.ceil(math.log(10) / (error**2)) # fixed confidence of 90%
	logging.info("taking {0} samples".format(nSamples))
	return ClusteringCoefficient().approxAvgLocal(G, nSamples)

def degreePowerLaw(G, dd=None):
	""" Check if a power law is a good fit for the degree distribution.

	"""
	if not dd:
		dd = degreeSequence(G)
	fit = powerlaw.Fit(dd)
	R, p = fit.distribution_compare("power_law", "exponential", normalized_ratio=True)
	gamma = fit.alpha
	return ((R > 0), R, gamma)

# def powerLawExponent_(G):
#     """ Estimate power law exponent as a linear regression line through a log-log plot
#     of the degree distribution"""
#     def logarithmic(data):
#     	return [(math.log(x) if x != 0 else 0) for x in data]

#     dd = degreeDistribution(G)
#     degrange = range(len(dd))
#     (slope,_,_,_,_) = stats.linregress(logarithmic(degrange), logarithmic(dd))
#     gamma = -1 * slope
#     return gamma

def degreeAssortativity(G):
	""" Returns the degree assortativity coefficient """
	return GraphProperties.degreeAssortativity(G, G.isWeighted())

def degeneracy(G):
	""" degeneracy of an undirected graph is defined as the largest k for which
	the graph has a non-empty k-core"""
	coreDec = CoreDecomposition(G)
	coreDec.run()
	return coreDec.maxCoreNumber()

def properties(G, settings):
	logging.info("[...] calculating properties")

	# size
	n, m = size(G)    # n and m

	logging.info("[...] determining degree distribution")
	# degrees
	degDist = GraphProperties.degreeDistribution(G)
	minDeg, maxDeg, avgDeg = degrees(G)

	plfit = degreePowerLaw(G)
	# number of isolated nodes
	isolates = degDist[0] if len(degDist) > 0 else None
	satellites = degDist[1] if len(degDist) > 1 else None

	# number of self-loops
	loops = len([(u, v) for (u,v) in G.edges() if (u == v)])

	# density
	dens = density(G)

	# community detection

	ncomPLP, modPLP = None, None
	ncomPLM, modPLM = None, None
	if settings["communities"]:
		logging.info("[...] detecting communities")
		# perform PLM community detection
		logging.info("[...] performing community detection: PLM")
		plm = community.PLM()
		print(plm)
		zetaPLM = plm.run(G)
		ncomPLM = zetaPLM.numberOfSubsets()
		modPLM = community.Modularity().getQuality(zetaPLM, G)

	# degree histogram

	labels, histo = None, None
	if settings["degreeHistogram"]:
		logging.info("[...] preparing degree histogram")
		histo = degDist
		(labels, histo) = compressHistogram(histo, nbins=25)

	# connected components
	nComponents, componentSizes = None, None
	if settings["components"]:
		nComponents, componentSizes = components(G)

	# diameter
	if settings["diameter"]:
		logging.info("[...] estimating diameter range")
		dia = Diameter.estimatedDiameterRange(G, error=0.1)
	else:
		dia = None

	# clustering
	avglcc = None
	if settings["clustering"]:
		logging.info("[...] approximating clustering coefficient")
		avglcc = clustering(G)

	# degree assortativity
	logging.info("[...] calculating degree assortativity coefficient")
	assort = degreeAssortativity(G)

	# degeneracy
	logging.info("[...] calculating degeneracy by k-core decomposition")
	degen = degeneracy(G)

	props = {
		 "name": G.getName(),
		 "n": n,
		 "m": m,
		 "directed": G.isDirected(),
		 "weighted": G.isWeighted(),
		 "minDeg": minDeg,
		 "maxDeg": maxDeg,
		 "avgDeg": avgDeg,
		 "plfit": plfit,
		 "avglcc": avglcc,
		 "degeneracy": degen,
		 "nComponents": nComponents,
		 "sizeLargestComponent": max(componentSizes.values()),
		 "dia": dia,
		 "isolates": isolates,
		 "loops": loops,
		 "ncomPLP": ncomPLP,
		 "modPLP": modPLP,
		 "ncomPLM": ncomPLM,
		 "modPLM": modPLM,
		 "dens": dens,
		 "assort": assort,
		 "histo": (labels, histo),
		 }

	return props

def overview(G, settings=collections.defaultdict(lambda: True)):
	"""
	Print an overview of important network properties to the terminal.
	"""
	props = properties(G, settings)
	basicProperties = [
		["nodes, edges", "{0}, {1}".format(props["n"], props["m"])],
		["directed?", "{0}".format(props["directed"])],
		["weighted?", "{0}".format(props["weighted"])],
		["isolated nodes", props["isolates"]],
		["self-loops", props["loops"]],
		["density", "{0:.6f}".format(props["dens"]) if props["dens"] else None],
		["clustering coefficient", "{0:.6f}".format(props["avglcc"]) if props["avglcc"] else None],
		["max. core number", props["degeneracy"]],
	]
	degreeProperties = [
		["min. degree", props["minDeg"]],
		["max. degree", props["maxDeg"]],
		["avg. degree", "{0:.6f}".format(props["avgDeg"])],
		["power law?, likelihood, gamma", "{0}, {1}, {2}".format(props["plfit"][0], "{0:.4f}".format(props["plfit"][1]), "{0:.4f}".format(props["plfit"][2]))],
		["degree assortativity", "{0:.4f}".format(props["assort"])],
	]
	pathStructure = [
		["connected components", props["nComponents"]],
		["size of largest component", props["sizeLargestComponent"]],
		["estimated diameter range", str(props["dia"])],
	]

	communityStructure = [
		["community detection (PLM)", "", ""],
		["", "communities", props["ncomPLM"]],
		["", "modularity", "{0:.6f}".format(props["modPLM"]) if props["modPLM"] else None],
	]

	print()
	print("Network Properties: {0}".format(G.getName()))
	print("==================")
	print("Basic Properties")
	print(tabulate.tabulate(basicProperties))
	print("Node Degree Properties")
	print(tabulate.tabulate(degreeProperties))
	print("Path Structure")
	print(tabulate.tabulate(pathStructure))
	#print("Miscellaneous")
	#print(tabulate.tabulate(miscProperties))
	print("Community Structure")
	print(tabulate.tabulate(communityStructure))
	print("Degree Distribution")
	print("-------------------")
	(labels, histo) = props["histo"]
	if labels and histo:
		termgraph.graph(labels, histo)

def compressHistogram(hist, nbins=20):
	""" Compress a histogram to a number of bins"""
	compressed = [None for i in range(nbins)]
	labels = [None for i in range(nbins)]
	nbinsprev = len(hist)
	binwidth = math.ceil(nbinsprev / nbins)
	for i in range(nbins):
		l = i * binwidth
		u = (i+1) * binwidth
		compressed[i] = sum(hist[l : u])
		labels[i] = "{0}-".format(l)
	return (labels, compressed)

def printDegreeHistogram(G, nbins=25):
	""" Prints a degree histogram as a bar chart to the terminal"""
	hist = GraphProperties.degreeDistribution(G)

	(labels, hist) = compressHistogram(hist, nbins)
	termgraph.graph(labels, hist)
<commit_msg>simplified properties.overview<commit_after># NetworKit native classes and functions
from _NetworKit import GraphProperties, ConnectedComponents, ParallelConnectedComponents, StronglyConnectedComponents, ClusteringCoefficient, Diameter, Eccentricity, CoreDecomposition

# other submodules
from . import community
from . import termgraph
from . import auxiliary
from . import nxadapter
from . import powerlaw

# other modules
import textwrap
import collections
import math
import logging

try:
	import networkx as nx
except ImportError:
	logging.warning("""WARNING: module 'networkx' not installed, which is required by some
						functions.""")
try:
	import tabulate
except ImportError:
	logging.warning("""WARNING: module 'tabulate' not installed, which is required by some
						functions. In order to install 'tabulate', Python 3.3 is required""")

try:
	from scipy import stats
except ImportError:
	logging.warning("""WARNING: module 'scipy' not installed, which is required by some
						functions.""")

########  PROPERTIES ########

def size(G):
	""" Return number of nodes and number of edges"""
	n = G.numberOfNodes()
	m = G.numberOfEdges()
	return (n, m)

def degrees(G):
	""" Return min/max/avg degree"""
	minMaxDeg = GraphProperties.minMaxDegree(G)
	avgDeg = GraphProperties.averageDegree(G)
	return (minMaxDeg[0], minMaxDeg[1], avgDeg)

def degreeDistribution(G):
	""" Return the degree distribution of a graph"""
	return GraphProperties.degreeDistribution(G)

def degreeSequence(G):
	""" Return the degree sequence of a graph"""
	return GraphProperties.degreeSequence(G)

def density(G):
	""" Return the density of the graph"""
	(n, m) = size(G)
	return (2 * m) / (n * (n-1))

def components(G):
	""" Find and analyze detected components.
		Returns the number of components and the sizes
		of each component. For more details use the
		ConnectedComponents class.
	"""
	logging.info("[...] finding connected components....")
	if G.isDirected():
		cc = StronglyConnectedComponents(G)
	else:
		cc = ConnectedComponents(G)
	cc.run()
	components = cc.getPartition()
	nComponents = components.numberOfSubsets()
	componentSizes = components.subsetSizeMap()
	return (nComponents, componentSizes)

def numberOfComponents(G):
	""" Find and number of components """
	logging.info("[...] finding connected components....")
	cc = ConnectedComponents()
	cc.run(G)
	nComponents = cc.numberOfComponents()
	return nComponents

def clustering(G, error=0.01):
	"""
		Returns approximate average local clustering coefficient
		The maximum error can be given as a parameter and determines
		the number of samples taken.

		for details see:
			Schank, Wagner: Approximating Clustering Coefficient and Transitivity
	"""
	nSamples = math.ceil(math.log(10) / (error**2)) # fixed confidence of 90%
	logging.info("taking {0} samples".format(nSamples))
	return ClusteringCoefficient().approxAvgLocal(G, nSamples)

def degreePowerLaw(G, dd=None):
	""" Check if a power law is a good fit for the degree distribution.

	"""
	if not dd:
		dd = degreeSequence(G)
	fit = powerlaw.Fit(dd)
	R, p = fit.distribution_compare("power_law", "exponential", normalized_ratio=True)
	gamma = fit.alpha
	return ((R > 0), R, gamma)

# def powerLawExponent_(G):
#     """ Estimate power law exponent as a linear regression line through a log-log plot
#     of the degree distribution"""
#     def logarithmic(data):
#     	return [(math.log(x) if x != 0 else 0) for x in data]

#     dd = degreeDistribution(G)
#     degrange = range(len(dd))
#     (slope,_,_,_,_) = stats.linregress(logarithmic(degrange), logarithmic(dd))
#     gamma = -1 * slope
#     return gamma

def degreeAssortativity(G):
	""" Returns the degree assortativity coefficient """
	return GraphProperties.degreeAssortativity(G, G.isWeighted())

def degeneracy(G):
	""" degeneracy of an undirected graph is defined as the largest k for which
	the graph has a non-empty k-core"""
	coreDec = CoreDecomposition(G)
	coreDec.run()
	return coreDec.maxCoreNumber()

def properties(G, settings):
	logging.info("[...] calculating properties")

	# size
	n, m = size(G)    # n and m

	logging.info("[...] determining degree distribution")
	# degrees
	degDist = GraphProperties.degreeDistribution(G)
	minDeg, maxDeg, avgDeg = degrees(G)

	plfit = degreePowerLaw(G)
	# number of isolated nodes
	isolates = degDist[0] if len(degDist) > 0 else None
	satellites = degDist[1] if len(degDist) > 1 else None

	# number of self-loops
	loops = len([(u, v) for (u,v) in G.edges() if (u == v)])

	# density
	dens = density(G)

	# community detection

	ncomPLP, modPLP = None, None
	ncomPLM, modPLM = None, None
	if settings["communities"]:
		logging.info("[...] detecting communities")
		# perform PLM community detection
		logging.info("[...] performing community detection: PLM")
		plm = community.PLM()
		print(plm)
		zetaPLM = plm.run(G)
		ncomPLM = zetaPLM.numberOfSubsets()
		modPLM = community.Modularity().getQuality(zetaPLM, G)

	# degree histogram

	labels, histo = None, None
	if settings["degreeHistogram"]:
		logging.info("[...] preparing degree histogram")
		histo = degDist
		(labels, histo) = compressHistogram(histo, nbins=25)

	# connected components
	nComponents, componentSizes = None, None
	if settings["components"]:
		nComponents, componentSizes = components(G)

	# diameter
	if settings["diameter"]:
		logging.info("[...] estimating diameter range")
		dia = Diameter.estimatedDiameterRange(G, error=0.1)
	else:
		dia = None

	# clustering
	avglcc = None
	if settings["clustering"]:
		logging.info("[...] approximating clustering coefficient")
		avglcc = clustering(G)

	# degree assortativity
	logging.info("[...] calculating degree assortativity coefficient")
	assort = degreeAssortativity(G)

	# degeneracy
	logging.info("[...] calculating degeneracy by k-core decomposition")
	degen = degeneracy(G)

	props = {
		 "name": G.getName(),
		 "n": n,
		 "m": m,
		 "directed": G.isDirected(),
		 "weighted": G.isWeighted(),
		 "minDeg": minDeg,
		 "maxDeg": maxDeg,
		 "avgDeg": avgDeg,
		 "plfit": plfit,
		 "avglcc": avglcc,
		 "degeneracy": degen,
		 "nComponents": nComponents,
		 "sizeLargestComponent": max(componentSizes.values()),
		 "dia": dia,
		 "isolates": isolates,
		 "loops": loops,
		 "ncomPLP": ncomPLP,
		 "modPLP": modPLP,
		 "ncomPLM": ncomPLM,
		 "modPLM": modPLM,
		 "dens": dens,
		 "assort": assort,
		 "histo": (labels, histo),
		 }

	return props

def overview(G, settings=collections.defaultdict(lambda: True)):
	"""
	Print an overview of important network properties to the terminal.
	"""
	props = properties(G, settings)
	basicProperties = [
		["nodes, edges", "{0}, {1}".format(props["n"], props["m"])],
		["directed?", "{0}".format(props["directed"])],
		["weighted?", "{0}".format(props["weighted"])],
		["isolated nodes", props["isolates"]],
		["self-loops", props["loops"]],
		["density", "{0:.6f}".format(props["dens"]) if props["dens"] else None],
		["clustering coefficient", "{0:.6f}".format(props["avglcc"]) if props["avglcc"] else None],
		["max. core number", props["degeneracy"]],
		["connected components", props["nComponents"]],
		["size of largest component", props["sizeLargestComponent"]],
		["estimated diameter range", str(props["dia"])],
	]
	degreeProperties = [
		["min./max. degree", "({0}, {1})".format(props["minDeg"], props["maxDeg"])],
		["avg. degree", "{0:.6f}".format(props["avgDeg"])],
		["power law?, likelihood, gamma", "{0}, {1}, {2}".format(props["plfit"][0], "{0:.4f}".format(props["plfit"][1]), "{0:.4f}".format(props["plfit"][2]))],
		["degree assortativity", "{0:.4f}".format(props["assort"])],
	]

	communityStructure = [
		["community detection (PLM)", "", ""],
		["", "communities", props["ncomPLM"]],
		["", "modularity", "{0:.6f}".format(props["modPLM"]) if props["modPLM"] else None],
	]

	print()
	print("Network Properties: {0}".format(G.getName()))
	print("==================")
	print("Basic Properties")
	print(tabulate.tabulate(basicProperties))
	print("Node Degree Properties")
	print(tabulate.tabulate(degreeProperties))
	#print("Miscellaneous")
	#print(tabulate.tabulate(miscProperties))
	print("Community Structure")
	print(tabulate.tabulate(communityStructure))
	print("Degree Distribution")
	print("-------------------")
	(labels, histo) = props["histo"]
	if labels and histo:
		termgraph.graph(labels, histo)

def compressHistogram(hist, nbins=20):
	""" Compress a histogram to a number of bins"""
	compressed = [None for i in range(nbins)]
	labels = [None for i in range(nbins)]
	nbinsprev = len(hist)
	binwidth = math.ceil(nbinsprev / nbins)
	for i in range(nbins):
		l = i * binwidth
		u = (i+1) * binwidth
		compressed[i] = sum(hist[l : u])
		labels[i] = "{0}-".format(l)
	return (labels, compressed)

def printDegreeHistogram(G, nbins=25):
	""" Prints a degree histogram as a bar chart to the terminal"""
	hist = GraphProperties.degreeDistribution(G)

	(labels, hist) = compressHistogram(hist, nbins)
	termgraph.graph(labels, hist)
<|endoftext|>