<commit_before>'''
All salt configuration loading and defaults should be in this module
'''

# Import python modules
import glob
import os
import socket
import logging
import time

# import third party libs
import yaml
try:
    yaml.Loader = yaml.CLoader
    yaml.Dumper = yaml.CDumper
except Exception:
    pass

# Import salt libs
import salt.crypt
import salt.loader
import salt.utils
import salt.pillar
from salt.exceptions import SaltClientError

log = logging.getLogger(__name__)

__dflt_log_datefmt = '%Y-%m-%d %H:%M:%S'
__dflt_log_fmt_console = '[%(levelname)-8s] %(message)s'
__dflt_log_fmt_logfile = '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'

def _validate_file_roots(file_roots):
    '''
    If the file_roots option has a key that is None then we will error out,
    just replace it with an empty list
    '''
    if not isinstance(file_roots, dict):
        log.warning('The file_roots parameter is not properly formatted,'
                    ' using defaults')
        return {'base': ['/srv/salt']}
    for env, dirs in list(file_roots.items()):
        if not isinstance(dirs, list) and not isinstance(dirs, tuple):
            file_roots[env] = []
    return file_roots

def _append_domain(opts):
    '''
    Append a domain to the existing id if it doesn't already exist
    '''
    # Domain already exists
    if opts['id'].endswith(opts['append_domain']):
        return opts['id']
    # Trailing dot should mean an FQDN that is terminated, leave it alone.
    if opts['id'].endswith('.'):
        return opts['id']
    return "{0[id]}.{0[append_domain]}".format(opts)

def _read_conf_file(path):
    with salt.utils.fopen(path, 'r') as conf_file:
        conf_opts = yaml.safe_load(conf_file.read()) or {}
        # allow using numeric ids: convert int to string
        if 'id' in conf_opts:
            conf_opts['id'] = str(conf_opts['id'])
        return conf_opts

def load_config(opts, path, env_var):
    '''
    Attempts to update ``opts`` dict by parsing either the file described by
    ``path`` or the environment variable described by ``env_var`` as YAML.
    '''
    if not path or not os.path.isfile(path):
        path = os.environ.get(env_var, path)
    # If the configuration file is missing, attempt to copy the template,
    # after removing the first header line.
    if not os.path.isfile(path):
        template = '{0}.template'.format(path)
        if os.path.isfile(template):
            import salt.utils  # Need to re-import, need to find out why
            with salt.utils.fopen(path, 'w') as out:
                with salt.utils.fopen(template, 'r') as f:
                    f.readline()  # skip first line
                    out.write(f.read())

    if os.path.isfile(path):
        try:
            opts.update(_read_conf_file(path))
            opts['conf_file'] = path
        except Exception as e:
            import salt.log
            msg = 'Error parsing configuration file: {0} - {1}'
            if salt.log.is_console_configured():
                log.warn(msg.format(path, e))
            else:
                print(msg.format(path, e))
    else:
        log.debug('Missing configuration file: {0}'.format(path))

def include_config(include, opts, orig_path, verbose):
    '''
    Parses extra configuration file(s) specified in an include list in the
    main config file.
    '''
    # Protect against empty option
    if not include:
        return opts

    if isinstance(include, str):
        include = [include]

    for path in include:
        if not os.path.isabs(path):
            path = os.path.join(os.path.dirname(orig_path), path)

        # Catch situation where user typos path in config; also warns for
        # empty include dir (which might be by design)
        if len(glob.glob(path)) == 0:
            msg = ('Warning parsing configuration file: "include" path/glob '
                   '"{0}" matches no files').format(path)
            if verbose: log.warn(msg.format(path))

        for fn_ in glob.glob(path):
            try:
                opts.update(_read_conf_file(fn_))
            except Exception as e:
                msg = 'Error parsing configuration file: {0} - {1}'
                log.warn(msg.format(fn_, e))
    return opts

def prepend_root_dir(opts, path_options):
    '''
    Prepends the options that represent filesystem paths with value of the
    'root_dir' option.
    '''
    root_dir = os.path.abspath(opts['root_dir'])
    for path_option in path_options:
        if path_option in opts:
            if opts[path_option].startswith(opts['root_dir']):
                opts[path_option] = opts[path_option][len(opts['root_dir']):]
            opts[path_option] = salt.utils.path_join(
                    root_dir,
                    opts[path_option])

def minion_config(path, check_dns=True):
    '''
    Reads in the minion configuration file and sets up special options
    '''
    opts = {'master': 'salt',
            'master_port': '4506',
            'master_finger': '',
            'user': 'root',
            'root_dir': '/',
            'pki_dir': '/etc/salt/pki/minion',
            'id': socket.getfqdn(),
            'cachedir': '/var/cache/salt/minion',
            'cache_jobs': False,
            'conf_file': path,
            'sock_dir': '/var/run/salt/minion',
            'backup_mode': '',
            'renderer': 'yaml_jinja',
            'failhard': False,
            'autoload_dynamic_modules': True,
            'environment': None,
            'state_top': 'top.sls',
            'startup_states': '',
            'sls_list': [],
            'top_file': '',
            'file_client': 'remote',
            'file_roots': {
                'base': ['/srv/salt'],
                },
            'pillar_roots': {
                'base': ['/srv/pillar'],
                },
            'hash_type': 'md5',
            'external_nodes': '',
            'disable_modules': [],
            'disable_returners': [],
            'module_dirs': [],
            'returner_dirs': [],
            'states_dirs': [],
            'render_dirs': [],
            'providers': {},
            'clean_dynamic_modules': True,
            'open_mode': False,
            'multiprocessing': True,
            'sub_timeout': 0,
            'ipc_mode': 'ipc',
            'tcp_pub_port': 4510,
            'tcp_pull_port': 4511,
            'log_file': '/var/log/salt/minion',
            'log_level': None,
            'log_level_logfile': None,
            'log_datefmt': __dflt_log_datefmt,
            'log_fmt_console': __dflt_log_fmt_console,
            'log_fmt_logfile': __dflt_log_fmt_logfile,
            'log_granular_levels': {},
            'test': False,
            'cython_enable': False,
            'state_verbose': True,
            'state_output': 'full',
            'acceptance_wait_time': 10,
            'dns_check': True,
            'verify_env': True,
            'grains': {},
            'permissive_pki_access': False,
            'default_include': 'minion.d/*.conf',
            'update_url': False,
            'update_restart_services': [],
            'retry_dns': 30,
            'recon_max': 30000,
            }

    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')

    load_config(opts, path, 'SALT_MINION_CONFIG')

    default_include = opts.get('default_include', [])
    include = opts.get('include', [])

    opts = include_config(default_include, opts, path, verbose=False)
    opts = include_config(include, opts, path, verbose=True)

    if 'append_domain' in opts:
        opts['id'] = _append_domain(opts)

    if check_dns:
        # Because I import salt.log bellow I need to re-import salt.utils here
        import salt.utils
        try:
            opts['master_ip'] = salt.utils.dns_check(opts['master'], True)
        except SaltClientError:
            if opts['retry_dns']:
                while True:
                    import salt.log
                    msg = ('Master hostname: {0} not found. Retrying in {1} '
                           'seconds').format(opts['master'], opts['retry_dns'])
                    if salt.log.is_console_configured():
                        log.warn(msg)
                    else:
                        print('WARNING: {0}'.format(msg))
                    time.sleep(opts['retry_dns'])
                    try:
                        opts['master_ip'] = salt.utils.dns_check(
                            opts['master'], True
                        )
                        break
                    except SaltClientError:
                        pass
            else:
                opts['master_ip'] = '127.0.0.1'
    else:
        opts['master_ip'] = '127.0.0.1'

    opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'],
                                                    port=opts['master_port'])

    # Enabling open mode requires that the value be set to True, and
    # nothing else!
    opts['open_mode'] = opts['open_mode'] is True

    # set up the extension_modules location from the cachedir
    opts['extension_modules'] = (
            opts.get('extension_modules') or
            os.path.join(opts['cachedir'], 'extmods')
            )

    # Prepend root_dir to other paths
    prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'sock_dir',
                            'key_logfile', 'extension_modules'])
    return opts

def master_config(path):
    '''
    Reads in the master configuration file and sets up default options
    '''
    opts = {'interface': '0.0.0.0',
            'publish_port': '4505',
            'user': 'root',
            'worker_threads': 5,
            'sock_dir': '/var/run/salt/master',
            'ret_port': '4506',
            'timeout': 5,
            'keep_jobs': 24,
            'root_dir': '/',
            'pki_dir': '/etc/salt/pki/master',
            'cachedir': '/var/cache/salt/master',
            'file_roots': {
                'base': ['/srv/salt'],
                },
            'master_roots': {
                'base': ['/srv/salt-master'],
                },
            'pillar_roots': {
                'base': ['/srv/pillar'],
                },
            'ext_pillar': [],
            # TODO - Set this to 2 by default in 0.10.5
            'pillar_version': 1,
            'pillar_opts': True,
            'syndic_master': '',
            'runner_dirs': [],
            'client_acl': {},
            'external_auth': {},
            'token_expire': 720,
            'file_buffer_size': 1048576,
            'max_open_files': 100000,
            'hash_type': 'md5',
            'conf_file': path,
            'open_mode': False,
            'auto_accept': False,
            'renderer': 'yaml_jinja',
            'failhard': False,
            'state_top': 'top.sls',
            'master_tops': {},
            'external_nodes': '',
            'order_masters': False,
            'job_cache': True,
            'ext_job_cache': '',
            'minion_data_cache': True,
            'log_file': '/var/log/salt/master',
            'log_level': None,
            'log_level_logfile': None,
            'log_datefmt': __dflt_log_datefmt,
            'log_fmt_console': __dflt_log_fmt_console,
            'log_fmt_logfile': __dflt_log_fmt_logfile,
            'log_granular_levels': {},
            'pidfile': '/var/run/salt-master.pid',
            'cluster_masters': [],
            'cluster_mode': 'paranoid',
            'range_server': 'range:80',
            'reactors': [],
            'serial': 'msgpack',
            'state_verbose': True,
            'state_output': 'full',
            'search': '',
            'search_index_interval': 3600,
            'nodegroups': {},
            'cython_enable': False,
            'key_logfile': '/var/log/salt/key',
            'verify_env': True,
            'permissive_pki_access': False,
            'default_include': 'master.d/*.conf',
    }

    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')

    load_config(opts, path, 'SALT_MASTER_CONFIG')

    default_include = opts.get('default_include', [])
    include = opts.get('include', [])

    opts = include_config(default_include, opts, path, verbose=False)
    opts = include_config(include, opts, path, verbose=True)

    opts['aes'] = salt.crypt.Crypticle.generate_key_string()

    opts['extension_modules'] = (
            opts.get('extension_modules') or
            os.path.join(opts['cachedir'], 'extmods')
            )
    opts['token_dir'] = os.path.join(opts['cachedir'], 'tokens')
    # Prepend root_dir to other paths
    prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file',
                            'sock_dir', 'key_logfile', 'extension_modules',
                            'autosign_file', 'token_dir'])

    # Enabling open mode requires that the value be set to True, and
    # nothing else!
    opts['open_mode'] = opts['open_mode'] is True
    opts['auto_accept'] = opts['auto_accept'] is True
    opts['file_roots'] = _validate_file_roots(opts['file_roots'])
    return opts

def client_config(path):
    '''
    Load in the configuration data needed for the LocalClient. This function
    searches for client specific configurations and adds them to the data from
    the master configuration.
    '''
    opts = {'token_file': os.path.expanduser('~/.salt_token')}
    opts.update(master_config(path))
    cpath = os.path.expanduser('~/.salt')
    load_config(opts, cpath, 'SALT_CLIENT_CONFIG')
    if 'token_file' in opts:
        opts['token_file'] = os.path.expanduser(opts['token_file'])
    if os.path.isfile(opts['token_file']):
        with salt.utils.fopen(opts['token_file']) as fp_:
            opts['token'] = fp_.read().strip()
    return opts
<commit_msg>Don't check DNS on local file client settings. Fixes #2731.<commit_after>'''
All salt configuration loading and defaults should be in this module
'''

# Import python modules
import glob
import os
import socket
import logging
import time

# import third party libs
import yaml
try:
    yaml.Loader = yaml.CLoader
    yaml.Dumper = yaml.CDumper
except Exception:
    pass

# Import salt libs
import salt.crypt
import salt.loader
import salt.utils
import salt.pillar
from salt.exceptions import SaltClientError

log = logging.getLogger(__name__)

__dflt_log_datefmt = '%Y-%m-%d %H:%M:%S'
__dflt_log_fmt_console = '[%(levelname)-8s] %(message)s'
__dflt_log_fmt_logfile = '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'

def _validate_file_roots(file_roots):
    '''
    If the file_roots option has a key that is None then we will error out,
    just replace it with an empty list
    '''
    if not isinstance(file_roots, dict):
        log.warning('The file_roots parameter is not properly formatted,'
                    ' using defaults')
        return {'base': ['/srv/salt']}
    for env, dirs in list(file_roots.items()):
        if not isinstance(dirs, list) and not isinstance(dirs, tuple):
            file_roots[env] = []
    return file_roots

def _append_domain(opts):
    '''
    Append a domain to the existing id if it doesn't already exist
    '''
    # Domain already exists
    if opts['id'].endswith(opts['append_domain']):
        return opts['id']
    # Trailing dot should mean an FQDN that is terminated, leave it alone.
    if opts['id'].endswith('.'):
        return opts['id']
    return "{0[id]}.{0[append_domain]}".format(opts)

def _read_conf_file(path):
    with salt.utils.fopen(path, 'r') as conf_file:
        conf_opts = yaml.safe_load(conf_file.read()) or {}
        # allow using numeric ids: convert int to string
        if 'id' in conf_opts:
            conf_opts['id'] = str(conf_opts['id'])
        return conf_opts

def load_config(opts, path, env_var):
    '''
    Attempts to update ``opts`` dict by parsing either the file described by
    ``path`` or the environment variable described by ``env_var`` as YAML.
    '''
    if not path or not os.path.isfile(path):
        path = os.environ.get(env_var, path)
    # If the configuration file is missing, attempt to copy the template,
    # after removing the first header line.
    if not os.path.isfile(path):
        template = '{0}.template'.format(path)
        if os.path.isfile(template):
            import salt.utils  # Need to re-import, need to find out why
            with salt.utils.fopen(path, 'w') as out:
                with salt.utils.fopen(template, 'r') as f:
                    f.readline()  # skip first line
                    out.write(f.read())

    if os.path.isfile(path):
        try:
            opts.update(_read_conf_file(path))
            opts['conf_file'] = path
        except Exception as e:
            import salt.log
            msg = 'Error parsing configuration file: {0} - {1}'
            if salt.log.is_console_configured():
                log.warn(msg.format(path, e))
            else:
                print(msg.format(path, e))
    else:
        log.debug('Missing configuration file: {0}'.format(path))

def include_config(include, opts, orig_path, verbose):
    '''
    Parses extra configuration file(s) specified in an include list in the
    main config file.
    '''
    # Protect against empty option
    if not include:
        return opts

    if isinstance(include, str):
        include = [include]

    for path in include:
        if not os.path.isabs(path):
            path = os.path.join(os.path.dirname(orig_path), path)

        # Catch situation where user typos path in config; also warns for
        # empty include dir (which might be by design)
        if len(glob.glob(path)) == 0:
            msg = ('Warning parsing configuration file: "include" path/glob '
                   '"{0}" matches no files').format(path)
            if verbose: log.warn(msg.format(path))

        for fn_ in glob.glob(path):
            try:
                opts.update(_read_conf_file(fn_))
            except Exception as e:
                msg = 'Error parsing configuration file: {0} - {1}'
                log.warn(msg.format(fn_, e))
    return opts

def prepend_root_dir(opts, path_options):
    '''
    Prepends the options that represent filesystem paths with value of the
    'root_dir' option.
    '''
    root_dir = os.path.abspath(opts['root_dir'])
    for path_option in path_options:
        if path_option in opts:
            if opts[path_option].startswith(opts['root_dir']):
                opts[path_option] = opts[path_option][len(opts['root_dir']):]
            opts[path_option] = salt.utils.path_join(
                    root_dir,
                    opts[path_option])

def minion_config(path, check_dns=True):
    '''
    Reads in the minion configuration file and sets up special options
    '''
    opts = {'master': 'salt',
            'master_port': '4506',
            'master_finger': '',
            'user': 'root',
            'root_dir': '/',
            'pki_dir': '/etc/salt/pki/minion',
            'id': socket.getfqdn(),
            'cachedir': '/var/cache/salt/minion',
            'cache_jobs': False,
            'conf_file': path,
            'sock_dir': '/var/run/salt/minion',
            'backup_mode': '',
            'renderer': 'yaml_jinja',
            'failhard': False,
            'autoload_dynamic_modules': True,
            'environment': None,
            'state_top': 'top.sls',
            'startup_states': '',
            'sls_list': [],
            'top_file': '',
            'file_client': 'remote',
            'file_roots': {
                'base': ['/srv/salt'],
                },
            'pillar_roots': {
                'base': ['/srv/pillar'],
                },
            'hash_type': 'md5',
            'external_nodes': '',
            'disable_modules': [],
            'disable_returners': [],
            'module_dirs': [],
            'returner_dirs': [],
            'states_dirs': [],
            'render_dirs': [],
            'providers': {},
            'clean_dynamic_modules': True,
            'open_mode': False,
            'multiprocessing': True,
            'sub_timeout': 0,
            'ipc_mode': 'ipc',
            'tcp_pub_port': 4510,
            'tcp_pull_port': 4511,
            'log_file': '/var/log/salt/minion',
            'log_level': None,
            'log_level_logfile': None,
            'log_datefmt': __dflt_log_datefmt,
            'log_fmt_console': __dflt_log_fmt_console,
            'log_fmt_logfile': __dflt_log_fmt_logfile,
            'log_granular_levels': {},
            'test': False,
            'cython_enable': False,
            'state_verbose': True,
            'state_output': 'full',
            'acceptance_wait_time': 10,
            'dns_check': True,
            'verify_env': True,
            'grains': {},
            'permissive_pki_access': False,
            'default_include': 'minion.d/*.conf',
            'update_url': False,
            'update_restart_services': [],
            'retry_dns': 30,
            'recon_max': 30000,
            }

    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')

    load_config(opts, path, 'SALT_MINION_CONFIG')

    default_include = opts.get('default_include', [])
    include = opts.get('include', [])

    opts = include_config(default_include, opts, path, verbose=False)
    opts = include_config(include, opts, path, verbose=True)

    if 'append_domain' in opts:
        opts['id'] = _append_domain(opts)

    if opts.get('file_client', 'remove') == 'local':
        check_dns = False

    if check_dns:
        # Because I import salt.log bellow I need to re-import salt.utils here
        import salt.utils
        try:
            opts['master_ip'] = salt.utils.dns_check(opts['master'], True)
        except SaltClientError:
            if opts['retry_dns']:
                while True:
                    import salt.log
                    msg = ('Master hostname: {0} not found. Retrying in {1} '
                           'seconds').format(opts['master'], opts['retry_dns'])
                    if salt.log.is_console_configured():
                        log.warn(msg)
                    else:
                        print('WARNING: {0}'.format(msg))
                    time.sleep(opts['retry_dns'])
                    try:
                        opts['master_ip'] = salt.utils.dns_check(
                            opts['master'], True
                        )
                        break
                    except SaltClientError:
                        pass
            else:
                opts['master_ip'] = '127.0.0.1'
    else:
        opts['master_ip'] = '127.0.0.1'

    opts['master_uri'] = 'tcp://{ip}:{port}'.format(ip=opts['master_ip'],
                                                    port=opts['master_port'])

    # Enabling open mode requires that the value be set to True, and
    # nothing else!
    opts['open_mode'] = opts['open_mode'] is True

    # set up the extension_modules location from the cachedir
    opts['extension_modules'] = (
            opts.get('extension_modules') or
            os.path.join(opts['cachedir'], 'extmods')
            )

    # Prepend root_dir to other paths
    prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file', 'sock_dir',
                            'key_logfile', 'extension_modules'])
    return opts

def master_config(path):
    '''
    Reads in the master configuration file and sets up default options
    '''
    opts = {'interface': '0.0.0.0',
            'publish_port': '4505',
            'user': 'root',
            'worker_threads': 5,
            'sock_dir': '/var/run/salt/master',
            'ret_port': '4506',
            'timeout': 5,
            'keep_jobs': 24,
            'root_dir': '/',
            'pki_dir': '/etc/salt/pki/master',
            'cachedir': '/var/cache/salt/master',
            'file_roots': {
                'base': ['/srv/salt'],
                },
            'master_roots': {
                'base': ['/srv/salt-master'],
                },
            'pillar_roots': {
                'base': ['/srv/pillar'],
                },
            'ext_pillar': [],
            # TODO - Set this to 2 by default in 0.10.5
            'pillar_version': 1,
            'pillar_opts': True,
            'syndic_master': '',
            'runner_dirs': [],
            'client_acl': {},
            'external_auth': {},
            'token_expire': 720,
            'file_buffer_size': 1048576,
            'max_open_files': 100000,
            'hash_type': 'md5',
            'conf_file': path,
            'open_mode': False,
            'auto_accept': False,
            'renderer': 'yaml_jinja',
            'failhard': False,
            'state_top': 'top.sls',
            'master_tops': {},
            'external_nodes': '',
            'order_masters': False,
            'job_cache': True,
            'ext_job_cache': '',
            'minion_data_cache': True,
            'log_file': '/var/log/salt/master',
            'log_level': None,
            'log_level_logfile': None,
            'log_datefmt': __dflt_log_datefmt,
            'log_fmt_console': __dflt_log_fmt_console,
            'log_fmt_logfile': __dflt_log_fmt_logfile,
            'log_granular_levels': {},
            'pidfile': '/var/run/salt-master.pid',
            'cluster_masters': [],
            'cluster_mode': 'paranoid',
            'range_server': 'range:80',
            'reactors': [],
            'serial': 'msgpack',
            'state_verbose': True,
            'state_output': 'full',
            'search': '',
            'search_index_interval': 3600,
            'nodegroups': {},
            'cython_enable': False,
            'key_logfile': '/var/log/salt/key',
            'verify_env': True,
            'permissive_pki_access': False,
            'default_include': 'master.d/*.conf',
    }

    if len(opts['sock_dir']) > len(opts['cachedir']) + 10:
        opts['sock_dir'] = os.path.join(opts['cachedir'], '.salt-unix')

    load_config(opts, path, 'SALT_MASTER_CONFIG')

    default_include = opts.get('default_include', [])
    include = opts.get('include', [])

    opts = include_config(default_include, opts, path, verbose=False)
    opts = include_config(include, opts, path, verbose=True)

    opts['aes'] = salt.crypt.Crypticle.generate_key_string()

    opts['extension_modules'] = (
            opts.get('extension_modules') or
            os.path.join(opts['cachedir'], 'extmods')
            )
    opts['token_dir'] = os.path.join(opts['cachedir'], 'tokens')
    # Prepend root_dir to other paths
    prepend_root_dir(opts, ['pki_dir', 'cachedir', 'log_file',
                            'sock_dir', 'key_logfile', 'extension_modules',
                            'autosign_file', 'token_dir'])

    # Enabling open mode requires that the value be set to True, and
    # nothing else!
    opts['open_mode'] = opts['open_mode'] is True
    opts['auto_accept'] = opts['auto_accept'] is True
    opts['file_roots'] = _validate_file_roots(opts['file_roots'])
    return opts

def client_config(path):
    '''
    Load in the configuration data needed for the LocalClient. This function
    searches for client specific configurations and adds them to the data from
    the master configuration.
    '''
    opts = {'token_file': os.path.expanduser('~/.salt_token')}
    opts.update(master_config(path))
    cpath = os.path.expanduser('~/.salt')
    load_config(opts, cpath, 'SALT_CLIENT_CONFIG')
    if 'token_file' in opts:
        opts['token_file'] = os.path.expanduser(opts['token_file'])
    if os.path.isfile(opts['token_file']):
        with salt.utils.fopen(opts['token_file']) as fp_:
            opts['token'] = fp_.read().strip()
    return opts
<|endoftext|>


<commit_before>'''
Routines to set up a minion
'''

# This module still needs package support, so that the functions dict
# returned can send back functions like: foo.bar.baz

# Import python libs
import os
import imp
import salt
import logging
import tempfile

# Import Salt libs
from salt.exceptions import LoaderError

log = logging.getLogger(__name__)
salt_base_path = os.path.dirname(salt.__file__)

def _create_loader(opts, ext_type, tag, ext_dirs=True, ext_type_dirs=None):
    '''
    Creates Loader instance

    Order of module_dirs:
        opts[ext_type_dirs],
        extension types,
        base types.
    '''
    ext_types = os.path.join(opts['extension_modules'], ext_type)
    sys_types = os.path.join(salt_base_path, ext_type)

    ext_type_types = []
    if ext_dirs:
        if ext_type_dirs is None:
            ext_type_dirs = '{0}_dirs'.format(tag)
        if ext_type_dirs in opts:
            ext_type_types.extend(opts[ext_type_dirs])

    module_dirs = ext_type_types + [ext_types, sys_types]
    return Loader(module_dirs, opts, tag)

def minion_mods(opts):
    '''
    Returns the minion modules
    '''
    load = _create_loader(opts, 'modules', 'module')
    functions = load.apply_introspection(load.gen_functions())
    if opts.get('providers', False):
        if isinstance(opts['providers'], dict):
            for mod, provider in opts['providers'].items():
                funcs = raw_mod(opts,
                        provider,
                        functions)
                if funcs:
                    for func in funcs:
                        f_key = '{0}{1}'.format(
                                mod,
                                func[func.rindex('.'):]
                                )
                        functions[f_key] = funcs[func]
    return functions

def raw_mod(opts, name, functions):
    '''
    Returns a single module loaded raw and bypassing the __virtual__ function
    '''
    load = _create_loader(opts, 'modules', 'rawmodule')
    return load.gen_module(name, functions)

def returners(opts):
    '''
    Returns the returner modules
    '''
    load = _create_loader(opts, 'returners', 'returner')
    return load.filter_func('returner')

def pillars(opts, functions):
    '''
    Returns the returner modules
    '''
    load = _create_loader(opts, 'pillar', 'pillar')
    pack = {'name': '__salt__',
            'value': functions}
    return load.filter_func('ext_pillar', pack)

def states(opts, functions):
    '''
    Returns the state modules
    '''
    load = _create_loader(opts, 'states', 'states')
    pack = {'name': '__salt__',
            'value': functions}
    return load.gen_functions(pack)

def render(opts, functions):
    '''
    Returns the render modules
    '''
    load = _create_loader(opts, 'renderers', 'render', ext_type_dirs='render_dirs')
    pack = {'name': '__salt__',
            'value': functions}
    rend = load.filter_func('render', pack)
    if opts['renderer'] not in rend:
        err = ('The renderer {0} is unavailable, this error is often because '
               'the needed software is unavailable'.format(opts['renderer']))
        log.critical(err)
        raise LoaderError(err)
    return rend

def grains(opts):
    '''
    Return the functions for the dynamic grains and the values for the static
    grains.
    '''
    if not 'grains' in opts:
        pre_opts = {}
        salt.config.load_config(
                pre_opts,
                opts['conf_file'],
                'SALT_MINION_CONFIG'
                )
        if 'include' in pre_opts:
            pre_opts = salt.config.include_config(
                    pre_opts,
                    opts['conf_file']
                    )
        if 'grains' in pre_opts:
            opts['grains'] = pre_opts['grains']
        else:
            opts['grains'] = {}
    load = _create_loader(opts, 'grains', 'grain', ext_dirs=False)
    grains = load.gen_grains()
    grains.update(opts['grains'])
    return grains

def call(fun, **kwargs):
    '''
    Directly call a function inside a loader directory
    '''
    args = kwargs.get('args', [])
    dirs = kwargs.get('dirs', [])
    module_dirs = [
        os.path.join(salt_base_path, 'modules'),
        ] + dirs
    load = Loader(module_dirs)
    return load.call(fun, args)

def runner(opts):
    '''
    Directly call a function inside a loader directory
    '''
    load = _create_loader(
            opts,
            'runners',
            'runner',
            ext_type_dirs='runner_dirs'
            )
    return load.gen_functions()

class Loader(object):
    '''
    Used to load in arbitrary modules from a directory, the Loader can
    also be used to only load specific functions from a directory, or to
    call modules in an arbitrary directory directly.
    '''
    def __init__(self, module_dirs, opts=dict(), tag='module'):
        self.module_dirs = module_dirs
        if '_' in tag:
            raise LoaderError('Cannot tag loader with an "_"')
        self.tag = tag
        if 'grains' in opts:
            self.grains = opts['grains']
        else:
            self.grains = {}
        if 'pillar' in opts:
            self.pillar = opts['pillar']
        else:
            self.pillar = {}
        self.opts = self.__prep_mod_opts(opts)

    def __prep_mod_opts(self, opts):
        '''
        Strip out of the opts any logger instance
        '''
        mod_opts = {}
        for key, val in opts.items():
            if key in ('logger', 'grains'):
                continue
            mod_opts[key] = val
        return mod_opts

    def get_docs(self, funcs, module=''):
        '''
        Return a dict containing all of the doc strings in the functions dict
        '''
        docs = {}
        for fun in funcs:
            if fun.startswith(module):
                docs[fun] = funcs[fun].__doc__
        return docs

    def call(self, fun, arg=list()):
        '''
        Call a function in the load path.
        '''
        name = fun[:fun.rindex('.')]
        try:
            fn_, path, desc = imp.find_module(name, self.module_dirs)
            mod = imp.load_module(name, fn_, path, desc)
        except ImportError:
            if self.opts.get('cython_enable', True) is True:
                # The module was not found, try to find a cython module
                try:
                    import pyximport
                    pyximport.install()

                    for mod_dir in self.module_dirs:
                        for fn_ in os.listdir(mod_dir):
                            if name == fn_[:fn_.rindex('.')]:
                                # Found it, load the mod and break the loop
                                mod = pyximport.load_module(
                                    name, os.path.join(mod_dir, fn_)
                                )
                                return getattr(
                                    mod, fun[fun.rindex('.') + 1:])(*arg)
                except ImportError:
                    log.info("Cython is enabled in options though it's not "
                             "present in the system path. Skipping Cython "
                             "modules.")
        return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)

    def gen_module(self, name, functions, pack=None):
        '''
        Load a single module and pack it with the functions passed
        '''
        full = ''
        mod = None
        for mod_dir in self.module_dirs:
            if not os.path.isabs(mod_dir):
                continue
            if not os.path.isdir(mod_dir):
                continue
            fn_ = os.path.join(mod_dir, name)
            for ext in ('.py', '.pyo', '.pyc', '.so'):
                full_test = '{0}{1}'.format(fn_, ext)
                if os.path.isfile(full_test):
                    full = full_test
        if not full:
            return None
        cython_enabled = False
        if self.opts.get('cython_enable', True) is True:
            try:
                import pyximport
                pyximport.install()
                cython_enabled = True
            except ImportError:
                log.info('Cython is enabled in the options but not present '
                         'in the system path. Skipping Cython modules.')
        try:
            if full.endswith('.pyx'):
                # If there's a name which ends in .pyx it means the above
                # cython_enabled is True. Continue...
                mod = pyximport.load_module(name, full, tempfile.gettempdir())
            else:
                fn_, path, desc = imp.find_module(name, self.module_dirs)
                mod = imp.load_module(
                        '{0}_{1}'.format(name, self.tag),
                        fn_,
                        path,
                        desc
                        )
        except ImportError as exc:
            log.debug(('Failed to import module {0}: {1}').format(name, exc))
            return mod
        except Exception as exc:
            log.warning(('Failed to import module {0}, this is due most'
                ' likely to a syntax error: {1}').format(name, exc))
            return mod
        if hasattr(mod, '__opts__'):
            mod.__opts__.update(self.opts)
        else:
            mod.__opts__ = self.opts

        mod.__grains__ = self.grains

        if pack:
            if isinstance(pack, list):
                for chunk in pack:
                    setattr(mod, chunk['name'], chunk['value'])
            else:
                setattr(mod, pack['name'], pack['value'])

        # Call a module's initialization method if it exists
        if hasattr(mod, '__init__'):
            if callable(mod.__init__):
                try:
                    mod.__init__()
                except TypeError:
                    pass
        funcs = {}
        for attr in dir(mod):
            if attr.startswith('_'):
                continue
            if callable(getattr(mod, attr)):
                func = getattr(mod, attr)
                if hasattr(func, '__bases__'):
                    if 'BaseException' in func.__bases__:
                        # the callable object is an exception, don't load it
                        continue
                funcs[
                        '{0}.{1}'.format(
                            mod.__name__[:mod.__name__.rindex('_')],
                            attr)
                        ] = func
                self._apply_outputter(func, mod)
        if not hasattr(mod, '__salt__'):
            mod.__salt__ = functions
        return funcs

    def gen_functions(self, pack=None, virtual_enable=True):
        '''
        Return a dict of functions found in the defined module_dirs
        '''
        names = {}
        modules = []
        funcs = {}
        disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))

        cython_enabled = False
        if self.opts.get('cython_enable', True) is True:
            try:
                import pyximport
                pyximport.install()
                cython_enabled = True
            except ImportError:
                log.info('Cython is enabled in the options but not present '
                         'in the system path. Skipping Cython modules.')
        for mod_dir in self.module_dirs:
            if not os.path.isabs(mod_dir):
                continue
            if not os.path.isdir(mod_dir):
                continue
            for fn_ in os.listdir(mod_dir):
                if fn_.startswith('_'):
                    continue
                if fn_.split('.')[0] in disable:
                    continue
                if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
                    or (cython_enabled and fn_.endswith('.pyx'))):
                    names[fn_[:fn_.rindex('.')]] = os.path.join(mod_dir, fn_)
        for name in names:
            try:
                if names[name].endswith('.pyx'):
                    # If there's a name which ends in .pyx it means the above
                    # cython_enabled is True. Continue...
                    mod = pyximport.load_module(
                            '{0}_{1}'.format(name, self.tag),
                            names[name],
                            tempfile.gettempdir())
                else:
                    fn_, path, desc = imp.find_module(name, self.module_dirs)
                    mod = imp.load_module(
                            '{0}_{1}'.format(name, self.tag),
                            fn_,
                            path,
                            desc
                            )
            except ImportError as exc:
                log.debug(('Failed to import module {0}, this is most likely'
                           ' NOT a problem: {1}').format(name, exc))
                continue
            except Exception as exc:
                log.warning(('Failed to import module {0}, this is due most'
                    ' likely to a syntax error: {1}').format(name, exc))
                continue
            modules.append(mod)
        for mod in modules:
            virtual = ''
            if hasattr(mod, '__opts__'):
                mod.__opts__.update(self.opts)
            else:
                mod.__opts__ = self.opts

            mod.__grains__ = self.grains
            mod.__pillar__ = self.pillar

            if pack:
                if isinstance(pack, list):
                    for chunk in pack:
                        setattr(mod, chunk['name'], chunk['value'])
                else:
                    setattr(mod, pack['name'], pack['value'])

            # Call a module's initialization method if it exists
            if hasattr(mod, '__init__'):
                if callable(mod.__init__):
                    try:
                        mod.__init__()
                    except TypeError:
                        pass

            if virtual_enable:
                if hasattr(mod, '__virtual__'):
                    if callable(mod.__virtual__):
                        virtual = mod.__virtual__()

            for attr in dir(mod):
                if attr.startswith('_'):
                    continue
                if callable(getattr(mod, attr)):
                    func = getattr(mod, attr)
                    if isinstance(func, type):
                        if any([
                            'Error' in func.__name__,
                            'Exception' in func.__name__]):
                            continue
                    if virtual:
                        funcs['{0}.{1}'.format(virtual, attr)] = func
                        self._apply_outputter(func, mod)
                    elif virtual is False:
                        pass
                    else:
                        funcs[
                                '{0}.{1}'.format(
                                    mod.__name__[:mod.__name__.rindex('_')],
                                    attr)
                                ] = func
                        self._apply_outputter(func, mod)
        for mod in modules:
            if not hasattr(mod, '__salt__'):
                mod.__salt__ = funcs
        return funcs

    def _apply_outputter(self, func, mod):
        '''
        Apply the __outputter__ variable to the functions
        '''
        if hasattr(mod, '__outputter__'):
            outp = mod.__outputter__
            if func.__name__ in outp:
                func.__outputter__ = outp[func.__name__]

    def apply_introspection(self, funcs):
        '''
        Pass in a function object returned from get_functions to load in
        introspection functions.
        '''
        funcs['sys.list_functions'] = lambda: self.list_funcs(funcs)
        funcs['sys.list_modules'] = lambda: self.list_modules(funcs)
        funcs['sys.doc'] = lambda module = '': self.get_docs(funcs, module)
        funcs['sys.reload_modules'] = lambda: True
        return funcs

    def list_funcs(self, funcs):
        '''
        List the functions
        '''
        return sorted(funcs)

    def list_modules(self, funcs):
        '''
        List the modules
        '''
        modules = set()
        for key in funcs:
            comps = key.split('.')
            if len(comps) < 2:
                continue
            modules.add(comps[0])
        return sorted(list(modules))

    def filter_func(self, name, pack=None):
        '''
        Filter a specific function out of the functions, this is used to load
        the returners for the salt minion
        '''
        funcs = {}
        gen = self.gen_functions(pack) if pack else self.gen_functions()
        for key, fun in gen.items():
            if key[key.index('.') + 1:] == name:
                funcs[key[:key.index('.')]] = fun
        return funcs

    def chop_mods(self):
        '''
        Chop off the module names so that the raw functions are exposed,
        used to generate the grains
        '''
        funcs = {}
        for key, fun in self.gen_functions().items():
            funcs[key[key.rindex('.')] + 1:] = fun
        return funcs

    def gen_grains(self):
        '''
        Read the grains directory and execute all of the public callable
        members. Then verify that the returns are python dict's and return
        a dict containing all of the returned values.
        '''
        grains = {}
        funcs = self.gen_functions()
        for key, fun in funcs.items():
            if not key[key.index('.') + 1:] == 'core':
                continue
            ret = fun()
            if not isinstance(ret, dict):
                continue
            grains.update(ret)
        for key, fun in funcs.items():
            if key[key.index('.') + 1:] == 'core':
                continue
            try:
                ret = fun()
            except Exception as exc:
                log.critical(('Failed to load grains defined in grain file '
                              '{0} in function {1}, error: {2}').format(
                                  key, fun, exc))
                continue
            if not isinstance(ret, dict):
                continue
            grains.update(ret)
        return grains
<commit_msg>Load functions from packages<commit_after>'''
Routines to set up a minion
'''

# This module still needs package support, so that the functions dict
# returned can send back functions like: foo.bar.baz

# Import python libs
import os
import imp
import salt
import logging
import tempfile

# Import Salt libs
from salt.exceptions import LoaderError

log = logging.getLogger(__name__)
salt_base_path = os.path.dirname(salt.__file__)

def _create_loader(opts, ext_type, tag, ext_dirs=True, ext_type_dirs=None):
    '''
    Creates Loader instance

    Order of module_dirs:
        opts[ext_type_dirs],
        extension types,
        base types.
    '''
    ext_types = os.path.join(opts['extension_modules'], ext_type)
    sys_types = os.path.join(salt_base_path, ext_type)

    ext_type_types = []
    if ext_dirs:
        if ext_type_dirs is None:
            ext_type_dirs = '{0}_dirs'.format(tag)
        if ext_type_dirs in opts:
            ext_type_types.extend(opts[ext_type_dirs])

    module_dirs = ext_type_types + [ext_types, sys_types]
    return Loader(module_dirs, opts, tag)

def minion_mods(opts):
    '''
    Returns the minion modules
    '''
    load = _create_loader(opts, 'modules', 'module')
    functions = load.apply_introspection(load.gen_functions())
    if opts.get('providers', False):
        if isinstance(opts['providers'], dict):
            for mod, provider in opts['providers'].items():
                funcs = raw_mod(opts,
                        provider,
                        functions)
                if funcs:
                    for func in funcs:
                        f_key = '{0}{1}'.format(
                                mod,
                                func[func.rindex('.'):]
                                )
                        functions[f_key] = funcs[func]
    return functions

def raw_mod(opts, name, functions):
    '''
    Returns a single module loaded raw and bypassing the __virtual__ function
    '''
    load = _create_loader(opts, 'modules', 'rawmodule')
    return load.gen_module(name, functions)

def returners(opts):
    '''
    Returns the returner modules
    '''
    load = _create_loader(opts, 'returners', 'returner')
    return load.filter_func('returner')

def pillars(opts, functions):
    '''
    Returns the returner modules
    '''
    load = _create_loader(opts, 'pillar', 'pillar')
    pack = {'name': '__salt__',
            'value': functions}
    return load.filter_func('ext_pillar', pack)

def states(opts, functions):
    '''
    Returns the state modules
    '''
    load = _create_loader(opts, 'states', 'states')
    pack = {'name': '__salt__',
            'value': functions}
    return load.gen_functions(pack)

def render(opts, functions):
    '''
    Returns the render modules
    '''
    load = _create_loader(opts, 'renderers', 'render', ext_type_dirs='render_dirs')
    pack = {'name': '__salt__',
            'value': functions}
    rend = load.filter_func('render', pack)
    if opts['renderer'] not in rend:
        err = ('The renderer {0} is unavailable, this error is often because '
               'the needed software is unavailable'.format(opts['renderer']))
        log.critical(err)
        raise LoaderError(err)
    return rend

def grains(opts):
    '''
    Return the functions for the dynamic grains and the values for the static
    grains.
    '''
    if not 'grains' in opts:
        pre_opts = {}
        salt.config.load_config(
                pre_opts,
                opts['conf_file'],
                'SALT_MINION_CONFIG'
                )
        if 'include' in pre_opts:
            pre_opts = salt.config.include_config(
                    pre_opts,
                    opts['conf_file']
                    )
        if 'grains' in pre_opts:
            opts['grains'] = pre_opts['grains']
        else:
            opts['grains'] = {}
    load = _create_loader(opts, 'grains', 'grain', ext_dirs=False)
    grains = load.gen_grains()
    grains.update(opts['grains'])
    return grains

def call(fun, **kwargs):
    '''
    Directly call a function inside a loader directory
    '''
    args = kwargs.get('args', [])
    dirs = kwargs.get('dirs', [])
    module_dirs = [
        os.path.join(salt_base_path, 'modules'),
        ] + dirs
    load = Loader(module_dirs)
    return load.call(fun, args)

def runner(opts):
    '''
    Directly call a function inside a loader directory
    '''
    load = _create_loader(
            opts,
            'runners',
            'runner',
            ext_type_dirs='runner_dirs'
            )
    return load.gen_functions()

class Loader(object):
    '''
    Used to load in arbitrary modules from a directory, the Loader can
    also be used to only load specific functions from a directory, or to
    call modules in an arbitrary directory directly.
    '''
    def __init__(self, module_dirs, opts=dict(), tag='module'):
        self.module_dirs = module_dirs
        if '_' in tag:
            raise LoaderError('Cannot tag loader with an "_"')
        self.tag = tag
        if 'grains' in opts:
            self.grains = opts['grains']
        else:
            self.grains = {}
        if 'pillar' in opts:
            self.pillar = opts['pillar']
        else:
            self.pillar = {}
        self.opts = self.__prep_mod_opts(opts)

    def __prep_mod_opts(self, opts):
        '''
        Strip out of the opts any logger instance
        '''
        mod_opts = {}
        for key, val in opts.items():
            if key in ('logger', 'grains'):
                continue
            mod_opts[key] = val
        return mod_opts

    def get_docs(self, funcs, module=''):
        '''
        Return a dict containing all of the doc strings in the functions dict
        '''
        docs = {}
        for fun in funcs:
            if fun.startswith(module):
                docs[fun] = funcs[fun].__doc__
        return docs

    def call(self, fun, arg=list()):
        '''
        Call a function in the load path.
        '''
        name = fun[:fun.rindex('.')]
        try:
            fn_, path, desc = imp.find_module(name, self.module_dirs)
            mod = imp.load_module(name, fn_, path, desc)
        except ImportError:
            if self.opts.get('cython_enable', True) is True:
                # The module was not found, try to find a cython module
                try:
                    import pyximport
                    pyximport.install()

                    for mod_dir in self.module_dirs:
                        for fn_ in os.listdir(mod_dir):
                            if name == fn_[:fn_.rindex('.')]:
                                # Found it, load the mod and break the loop
                                mod = pyximport.load_module(
                                    name, os.path.join(mod_dir, fn_)
                                )
                                return getattr(
                                    mod, fun[fun.rindex('.') + 1:])(*arg)
                except ImportError:
                    log.info("Cython is enabled in options though it's not "
                             "present in the system path. Skipping Cython "
                             "modules.")
        return getattr(mod, fun[fun.rindex('.') + 1:])(*arg)

    def gen_module(self, name, functions, pack=None):
        '''
        Load a single module and pack it with the functions passed
        '''
        full = ''
        mod = None
        for mod_dir in self.module_dirs:
            if not os.path.isabs(mod_dir):
                continue
            if not os.path.isdir(mod_dir):
                continue
            fn_ = os.path.join(mod_dir, name)
            if os.path.isdir(fn_):
                full = fn_
            else:
                for ext in ('.py', '.pyo', '.pyc', '.so'):
                    full_test = '{0}{1}'.format(fn_, ext)
                    if os.path.isfile(full_test):
                        full = full_test
        if not full:
            return None
        cython_enabled = False
        if self.opts.get('cython_enable', True) is True:
            try:
                import pyximport
                pyximport.install()
                cython_enabled = True
            except ImportError:
                log.info('Cython is enabled in the options but not present '
                         'in the system path. Skipping Cython modules.')
        try:
            if full.endswith('.pyx'):
                # If there's a name which ends in .pyx it means the above
                # cython_enabled is True. Continue...
                mod = pyximport.load_module(name, full, tempfile.gettempdir())
            else:
                fn_, path, desc = imp.find_module(name, self.module_dirs)
                mod = imp.load_module(
                        '{0}_{1}'.format(name, self.tag),
                        fn_,
                        path,
                        desc
                        )
        except ImportError as exc:
            log.debug(('Failed to import module {0}: {1}').format(name, exc))
            return mod
        except Exception as exc:
            log.warning(('Failed to import module {0}, this is due most'
                ' likely to a syntax error: {1}').format(name, exc))
            return mod
        if hasattr(mod, '__opts__'):
            mod.__opts__.update(self.opts)
        else:
            mod.__opts__ = self.opts

        mod.__grains__ = self.grains

        if pack:
            if isinstance(pack, list):
                for chunk in pack:
                    setattr(mod, chunk['name'], chunk['value'])
            else:
                setattr(mod, pack['name'], pack['value'])

        # Call a module's initialization method if it exists
        if hasattr(mod, '__init__'):
            if callable(mod.__init__):
                try:
                    mod.__init__()
                except TypeError:
                    pass
        funcs = {}
        for attr in dir(mod):
            if attr.startswith('_'):
                continue
            if callable(getattr(mod, attr)):
                func = getattr(mod, attr)
                if hasattr(func, '__bases__'):
                    if 'BaseException' in func.__bases__:
                        # the callable object is an exception, don't load it
                        continue
                funcs[
                        '{0}.{1}'.format(
                            mod.__name__[:mod.__name__.rindex('_')],
                            attr)
                        ] = func
                self._apply_outputter(func, mod)
        if not hasattr(mod, '__salt__'):
            mod.__salt__ = functions
        return funcs

    def gen_functions(self, pack=None, virtual_enable=True):
        '''
        Return a dict of functions found in the defined module_dirs
        '''
        names = {}
        modules = []
        funcs = {}
        disable = set(self.opts.get('disable_{0}s'.format(self.tag), []))

        cython_enabled = False
        if self.opts.get('cython_enable', True) is True:
            try:
                import pyximport
                pyximport.install()
                cython_enabled = True
            except ImportError:
                log.info('Cython is enabled in the options but not present '
                         'in the system path. Skipping Cython modules.')
        for mod_dir in self.module_dirs:
            if not os.path.isabs(mod_dir):
                continue
            if not os.path.isdir(mod_dir):
                continue
            for fn_ in os.listdir(mod_dir):
                if fn_.startswith('_'):
                    continue
                if fn_.split('.')[0] in disable:
                    continue
                if (fn_.endswith(('.py', '.pyc', '.pyo', '.so'))
                    or (cython_enabled and fn_.endswith('.pyx'))
                    or os.path.isdir(fn_)):
                    extpos = fn_.rfind('.')
                    if extpos > 0:
                        _name = fn_[:extpos]
                    else:
                        _name = fn_
                    names[_name] = os.path.join(mod_dir, fn_)
        for name in names:
            try:
                if names[name].endswith('.pyx'):
                    # If there's a name which ends in .pyx it means the above
                    # cython_enabled is True. Continue...
                    mod = pyximport.load_module(
                            '{0}_{1}'.format(name, self.tag),
                            names[name],
                            tempfile.gettempdir())
                else:
                    fn_, path, desc = imp.find_module(name, self.module_dirs)
                    mod = imp.load_module(
                            '{0}_{1}'.format(name, self.tag),
                            fn_,
                            path,
                            desc
                            )
            except ImportError as exc:
                log.debug(('Failed to import module {0}, this is most likely'
                           ' NOT a problem: {1}').format(name, exc))
                continue
            except Exception as exc:
                log.warning(('Failed to import module {0}, this is due most'
                    ' likely to a syntax error: {1}').format(name, exc))
                continue
            modules.append(mod)
        for mod in modules:
            virtual = ''
            if hasattr(mod, '__opts__'):
                mod.__opts__.update(self.opts)
            else:
                mod.__opts__ = self.opts

            mod.__grains__ = self.grains
            mod.__pillar__ = self.pillar

            if pack:
                if isinstance(pack, list):
                    for chunk in pack:
                        setattr(mod, chunk['name'], chunk['value'])
                else:
                    setattr(mod, pack['name'], pack['value'])

            # Call a module's initialization method if it exists
            if hasattr(mod, '__init__'):
                if callable(mod.__init__):
                    try:
                        mod.__init__()
                    except TypeError:
                        pass

            if virtual_enable:
                if hasattr(mod, '__virtual__'):
                    if callable(mod.__virtual__):
                        virtual = mod.__virtual__()

            for attr in dir(mod):
                if attr.startswith('_'):
                    continue
                if callable(getattr(mod, attr)):
                    func = getattr(mod, attr)
                    if isinstance(func, type):
                        if any([
                            'Error' in func.__name__,
                            'Exception' in func.__name__]):
                            continue
                    if virtual:
                        funcs['{0}.{1}'.format(virtual, attr)] = func
                        self._apply_outputter(func, mod)
                    elif virtual is False:
                        pass
                    else:
                        funcs[
                                '{0}.{1}'.format(
                                    mod.__name__[:mod.__name__.rindex('_')],
                                    attr)
                                ] = func
                        self._apply_outputter(func, mod)
        for mod in modules:
            if not hasattr(mod, '__salt__'):
                mod.__salt__ = funcs
        return funcs

    def _apply_outputter(self, func, mod):
        '''
        Apply the __outputter__ variable to the functions
        '''
        if hasattr(mod, '__outputter__'):
            outp = mod.__outputter__
            if func.__name__ in outp:
                func.__outputter__ = outp[func.__name__]

    def apply_introspection(self, funcs):
        '''
        Pass in a function object returned from get_functions to load in
        introspection functions.
        '''
        funcs['sys.list_functions'] = lambda: self.list_funcs(funcs)
        funcs['sys.list_modules'] = lambda: self.list_modules(funcs)
        funcs['sys.doc'] = lambda module = '': self.get_docs(funcs, module)
        funcs['sys.reload_modules'] = lambda: True
        return funcs

    def list_funcs(self, funcs):
        '''
        List the functions
        '''
        return sorted(funcs)

    def list_modules(self, funcs):
        '''
        List the modules
        '''
        modules = set()
        for key in funcs:
            comps = key.split('.')
            if len(comps) < 2:
                continue
            modules.add(comps[0])
        return sorted(list(modules))

    def filter_func(self, name, pack=None):
        '''
        Filter a specific function out of the functions, this is used to load
        the returners for the salt minion
        '''
        funcs = {}
        gen = self.gen_functions(pack) if pack else self.gen_functions()
        for key, fun in gen.items():
            if key[key.index('.') + 1:] == name:
                funcs[key[:key.index('.')]] = fun
        return funcs

    def chop_mods(self):
        '''
        Chop off the module names so that the raw functions are exposed,
        used to generate the grains
        '''
        funcs = {}
        for key, fun in self.gen_functions().items():
            funcs[key[key.rindex('.')] + 1:] = fun
        return funcs

    def gen_grains(self):
        '''
        Read the grains directory and execute all of the public callable
        members. Then verify that the returns are python dict's and return
        a dict containing all of the returned values.
        '''
        grains = {}
        funcs = self.gen_functions()
        for key, fun in funcs.items():
            if not key[key.index('.') + 1:] == 'core':
                continue
            ret = fun()
            if not isinstance(ret, dict):
                continue
            grains.update(ret)
        for key, fun in funcs.items():
            if key[key.index('.') + 1:] == 'core':
                continue
            try:
                ret = fun()
            except Exception as exc:
                log.critical(('Failed to load grains defined in grain file '
                              '{0} in function {1}, error: {2}').format(
                                  key, fun, exc))
                continue
            if not isinstance(ret, dict):
                continue
            grains.update(ret)
        return grains
<|endoftext|>


<commit_before>#!/usr/bin/env python
# PYTHON_ARGCOMPLETE_OK

import os.path
import string
import sys
import time
import re
import argparse
import argcomplete
import json
from collections import OrderedDict
from sysexecute import *
from banksync_common import *

__version__ = "0.7.1"

# --------------------------------------------------------------------------------------------------------------------------
# Defines
# --------------------------------------------------------------------------------------------------------------------------

tryOrder = ["sha", "UnixTimeStamp"]
defaultSyncPointBranchName = "syncPoint"

# --------------------------------------------------------------------------------------------------------------------------
# Parse the arguments
# --------------------------------------------------------------------------------------------------------------------------

sync_commands = ['sync', 'createSyncPoint', 'generateSyncFile', 'bisect']
git_commands = ['gitclone', 'gitreset', 'gitlog', 'gitstatus', 'gitbranch', 'gitcheckout', 'gitcommit', 'gitdiff', 'gitfecth', 'gitpush', 'gitpull', 'gitprune', 'gitgc', 'gitfsck']
commands = sync_commands + git_commands
matchingOpts = ['shaOnly', 'timestamp', 'closetimestamp']

mainDescription = 'execute operations across a collection of git repositories.'
bodyDescription = stringWithVars(
'''Utility to checkout or create a synchronized state across a collection of
repositories. This utility works on a syncfile. It is a more general way to
handle sub-repositories / submodules. It is intended to be less brittle than
traditional ways to specify submodules by allowing some looseness / decoupling.

It can also be used to issue a normal git command to every repository specified in the syncfile.
Current git commands allowed are {git_commands} (although adding others is a quick script change)

The options, eg the --syncfile option and the --cwd  option, can be specified in a standard ini
config file `bankconfig.ini` so they do not need to be specified each time on the command line.

Example usage:

  bank sync --syncfile syncfile.wl

This would checkout / update the repos given in the syncfile to the states given in the syncfile.

  bank sync --syncfile syncfile.wl --cwd ../other/dir

This would checkout / update the repos given in the syncfile to the states given in the syncfile
(but the path to the repos are prefixed by the value of cwd).

  bank createSyncPoint --syncfile syncfile.wl

This would alter the revisions stored in the syncfile.wl to match the current revisions of the referenced repositories.

  bank generateSyncFile --syncfile syncfile.wl repo1 repo2 ... repoN

This would generate or overwrite the syncfile.wl to contain sync points for the current states of repo1 repo2 ... repoN

  bank gitstatus --syncfile syncfile.wl

Perform gitstatus and list the results on each of the repositories specified in the syncfile.wl

  bank sync

Use the syncfile specified in the  and list the results on each of the repositories specified in the syncfile.wl
''')

bodyDescription = wrapParagraphs(bodyDescription)

def parseArguments():
    parser = argparse.ArgumentParser(description=mainDescription, epilog=bodyDescription, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("command", metavar="ACTION", nargs='?', help=stringWithVars("perform one of {sync_commands} on a syncfile, or one of {git_commands} on all the repos in the bank."), choices=commands, default='gitstatus')
    parser.add_argument("--syncfile", metavar="SYNCFILE", help="the path to the syncfile", default='Automatic')
    parser.add_argument("--cwd", metavar="CWD", help="prefix / change the working directory for the repos in the sync file", default="Automatic")
    parser.add_argument("--matching", metavar="MATCH", help=stringWithVars('specify how we can recognize a revision "match": {matchingOpts}'), choices=matchingOpts, default="Automatic")
    parser.add_argument("--verbosity", metavar="NUM", help="Specify the level of feedback detail for the install", default="Automatic")
    parser.add_argument('--dryRun',dest='dryRun',action='store_true', help="Print what would happen instead of executing the deploy")
    parser.add_argument('--version',dest='version',action='store_true', help="Show the version number of the banksync tool")
    parser.set_defaults(dryrun=False)
    parser.set_defaults(version=False)
    argcomplete.autocomplete(parser)

    def printVersionAndExit():
        printWithVars1("banksync {__version__}. Author: Jason F Harris.\nhttps://github.com/jasonfharris/banksync")
        sys.exit(0)

    if len(sys.argv)==1:
        parser.print_help()
        sys.exit(1)

    if len(sys.argv)==2 and sys.argv[1] == '--version':
        printVersionAndExit()

    args, remainingArgs = parser.parse_known_args()
    remainingArgs = [correctlyQuoteArg(arg) for arg in remainingArgs]
    if args.version:
        printVersionAndExit()

    return args, remainingArgs

# --------------------------------------------------------------------------------------------------------------------------
# command "sync"
# --------------------------------------------------------------------------------------------------------------------------

def commandSync():
    checkForSyncRepo(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    allFound = True

    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        repoString = paddedRepoName(repoName,syncDict)
        if not checkForRepo(repoString, absRepoPath):
            allFound = False
            continue

        found = False
        for method in tryOrder:
            if found:
                break

            if (method == "sha") and ("sha" in repoInfo):
                hash = repoInfo["sha"]
                res = gitCommand("git checkout -B {defaultSyncPointBranchName} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                if res["code"] == 0:
                    revNum = getRevNumber(hash, absRepoPath)
                    printWithVars2("{repoString}: successfully checked out revision by {method}: {hash} (revision number {revNum})")
                    found = True
                    break
                printWithVars3("{repoString}: failed to check out revision by {method}: {hash}")

            if (method == "UnixTimeStamp") and ("UnixTimeStamp" in repoInfo):
                if (matching == 'timestamp') or (matching == 'closetimestamp'):
                    ts = repoInfo["UnixTimeStamp"]
                    date = dateFromTimeStamp(ts)
                    res = gitCommand("git log --all --format=format:'\"%at\" : \"%H\",'", 4, cwd=absRepoPath, raiseOnFailure=True, verbosity=verbosity, permitShowingStdOut=False, permitShowingStdErr=False)
                    shaHash = 0
                    if res["code"] == 0:
                        timestampsToShas = json.loads('{'+res["stdout"][0:-1]+'}')
                        if (ts in timestampsToShas):
                            if (matching == 'timestamp') or (matching == 'closetimestamp'):
                                hash = timestampsToShas[ts]
                                branch=defaultSyncPointBranchName
                                res = gitCommand("git checkout -B {branch} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                                if res["code"] == 0:
                                    revNum = getRevNumber(hash, absRepoPath)
                                    printWithVars2("{repoString}: successfully checked out revision by {method}: {ts} ({date}) {hash} (revision number {revNum})")
                                    found = True
                                    break
                        else:
                            if matching == 'closetimestamp':
                                closestTimestamp = min(timestampsToShas, key=lambda x:abs(int(x)-int(ts)))
                                closestDate = dateFromTimeStamp(closestTimestamp)
                                hash = timestampsToShas[closestTimestamp]
                                branch=defaultSyncPointBranchName
                                res = gitCommand("git checkout -B {branch} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                                if res["code"] == 0:
                                    revNum = getRevNumber(hash, absRepoPath)
                                    printWithVars2("{repoString}: warning checking out revision by closest timestamp.", "red")
                                    printWithVars2("       requested {method}: {ts} ({date})")
                                    printWithVars2("       used      {method}: {closestTimestamp} ({closestDate}) {hash} (revision number {revNum})")
                                    found = True
                                    break

                    printWithVars3("{repoString}: failed to check out revision by {method}: {ts} {date}")

        if not found:
            allFound = False
            printWithVars2("{repoString}: failed to check out specified revision by any method.")
    if allFound:
        print colored("success! all repos checked out to the specified sync state.", 'green')
    else:
        print colored("failure! not all repos checked out to the specified sync state.", 'red')
        sys.exit(1)

# --------------------------------------------------------------------------------------------------------------------------
# command "createSyncPoint"
# --------------------------------------------------------------------------------------------------------------------------

def commandSyncCreateSyncPoint():
    checkForSyncRepo(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    newSyncDict = OrderedDict(syncDict)
    anyFailures = False
    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        repoString = paddedRepoName(repoName,syncDict)
        if not checkForRepo(repoString, absRepoPath):
            anyFailures = True
            continue
        (worked, newRepoInfo) = dictFromCurrentRepoState(repoInfo["path"], cwd=cwd, verbosity=verbosity)
        if worked:
            hash = newRepoInfo["sha"]
            date = newRepoInfo["date"]
            printWithVars2("{repoName}: recording bank sync state of {hash}, {date}.")
        else:
            printWithVars2("failure! not able to get the status of {repoName} at {absRepoPath}", 'red')
            anyFailures = True
        newSyncDict[repoName] = newRepoInfo

    writeDictToSyncFile(syncFilePath, newSyncDict)

    if anyFailures:
        printWithVars1("failure! not all constituent repos had their state recorded.", 'red')
        sys.exit(1)
    else:
        printWithVars1("success! all constituent repos had their state recorded.", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# command "generateSyncFile"
# --------------------------------------------------------------------------------------------------------------------------

def commandGenerateSyncFile():
    checkForSyncRepoDir(syncFilePath, existing = False)
    newSyncDict = OrderedDict()
    anyFailures = False
    for repo in remainingArgs:
        absRepoPath = getAbsRepoPath(repo, cwd)
        repoName = os.path.basename(absRepoPath)
        if not checkForRepo(repoName, absRepoPath):
            anyFailures = True
            continue
        (worked, newRepoInfo) = dictFromCurrentRepoState(repo, cwd=cwd, verbosity=verbosity)
        if worked:
            hash = newRepoInfo["sha"]
            date = newRepoInfo["date"]
            printWithVars2("{repoName}: recording bank sync state of {hash}, {date}.")
        else:
            printWithVars2("failure! not able to get the status of {repoName} at {absRepoPath}", 'red')
            anyFailures = True
        newSyncDict[repoName] = newRepoInfo

    writeDictToSyncFile(syncFilePath, newSyncDict)

    if anyFailures:
        printWithVars1("failure! not all constituent repos had their state recorded.", 'red')
        sys.exit(1)
    else:
        printWithVars1("success! all constituent repos had their state recorded.", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# command "bisect"
# --------------------------------------------------------------------------------------------------------------------------

def commandBisect():
    bisectCmd = "git bisect " + " ".join(remainingArgs)
    res = gitCommand(bisectCmd, 2, cwd=syncRepoPath, verbosity=verbosity);
    commandSync()

# --------------------------------------------------------------------------------------------------------------------------
# a git command
# --------------------------------------------------------------------------------------------------------------------------

def distributeGitCommand(command):
    cmd = command[3:]
    gitCmd = "git " + command[3:] + " " + " ".join(remainingArgs)
    gitCmd = gitCmd.strip()
    checkForSyncRepoDir(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    anyFailures = False
    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        if not checkForRepo(repoName, absRepoPath):
            anyFailures = True
            continue
        res = gitCommand(gitCmd, 2, cwd=absRepoPath, verbosity=verbosity);
    if anyFailures:
        printWithVars1("failure! not all constituent repos present.", 'red')
        sys.exit(1)
    else:
        printWithVars1("all constituent repos issued git command '{gitCmd}'", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# dispatch command
# --------------------------------------------------------------------------------------------------------------------------

def dispatchCommand(command):
    #from pudb import set_trace; set_trace()
    if command == "sync":
        commandSync()
    if command == "createSyncPoint":
        commandSyncCreateSyncPoint()
    if command == "generateSyncFile":
        commandGenerateSyncFile()
    if command == "bisect":
        commandBisect()
    if command in git_commands:
        distributeGitCommand(command)

# --------------------------------------------------------------------------------------------------------------------------
# Extract and clean the argument parameters
# --------------------------------------------------------------------------------------------------------------------------

def main():
    global remainingArgs, syncFilePath, syncRepoPath, cwd, matching, verbosity
    args, remainingArgs = parseArguments()

    # Get the config file path
    if args.cwd != "Automatic":
        configFile = os.path.abspath(os.path.join(args.cwd, 'bankconfig.ini'))
    else:
        configFile = os.path.abspath('bankconfig.ini')

    command = args.command
    syncFilePath = getSetting(args.syncfile, configFile, 'syncfile', 'syncfile.wl')
    syncRepoPath = os.path.dirname(os.path.abspath(syncFilePath))
    cwd = getSetting(args.cwd, configFile, 'cwd', '.')
    matching = getSetting(args.matching, configFile, 'matching', 'closetimestamp')
    verbosity= getSetting(args.verbosity, configFile, 'verbosity', 2)
    set_defaults('verbosity', verbosity)
    set_defaults('dryRun', args.dryRun)

    dispatchCommand(command)

if __name__ == '__main__':
    main()<commit_msg>- Color the output a tiny bit more to indicate success and failure.<commit_after>#!/usr/bin/env python
# PYTHON_ARGCOMPLETE_OK

import os.path
import string
import sys
import time
import re
import argparse
import argcomplete
import json
from collections import OrderedDict
from sysexecute import *
from banksync_common import *

__version__ = "0.7.1"

# --------------------------------------------------------------------------------------------------------------------------
# Defines
# --------------------------------------------------------------------------------------------------------------------------

tryOrder = ["sha", "UnixTimeStamp"]
defaultSyncPointBranchName = "syncPoint"

# --------------------------------------------------------------------------------------------------------------------------
# Parse the arguments
# --------------------------------------------------------------------------------------------------------------------------

sync_commands = ['sync', 'createSyncPoint', 'generateSyncFile', 'bisect']
git_commands = ['gitclone', 'gitreset', 'gitlog', 'gitstatus', 'gitbranch', 'gitcheckout', 'gitcommit', 'gitdiff', 'gitfecth', 'gitpush', 'gitpull', 'gitprune', 'gitgc', 'gitfsck']
commands = sync_commands + git_commands
matchingOpts = ['shaOnly', 'timestamp', 'closetimestamp']

mainDescription = 'execute operations across a collection of git repositories.'
bodyDescription = stringWithVars(
'''Utility to checkout or create a synchronized state across a collection of
repositories. This utility works on a syncfile. It is a more general way to
handle sub-repositories / submodules. It is intended to be less brittle than
traditional ways to specify submodules by allowing some looseness / decoupling.

It can also be used to issue a normal git command to every repository specified in the syncfile.
Current git commands allowed are {git_commands} (although adding others is a quick script change)

The options, eg the --syncfile option and the --cwd  option, can be specified in a standard ini
config file `bankconfig.ini` so they do not need to be specified each time on the command line.

Example usage:

  bank sync --syncfile syncfile.wl

This would checkout / update the repos given in the syncfile to the states given in the syncfile.

  bank sync --syncfile syncfile.wl --cwd ../other/dir

This would checkout / update the repos given in the syncfile to the states given in the syncfile
(but the path to the repos are prefixed by the value of cwd).

  bank createSyncPoint --syncfile syncfile.wl

This would alter the revisions stored in the syncfile.wl to match the current revisions of the referenced repositories.

  bank generateSyncFile --syncfile syncfile.wl repo1 repo2 ... repoN

This would generate or overwrite the syncfile.wl to contain sync points for the current states of repo1 repo2 ... repoN

  bank gitstatus --syncfile syncfile.wl

Perform gitstatus and list the results on each of the repositories specified in the syncfile.wl

  bank sync

Use the syncfile specified in the  and list the results on each of the repositories specified in the syncfile.wl
''')

bodyDescription = wrapParagraphs(bodyDescription)

def parseArguments():
    parser = argparse.ArgumentParser(description=mainDescription, epilog=bodyDescription, formatter_class=argparse.RawDescriptionHelpFormatter)
    parser.add_argument("command", metavar="ACTION", nargs='?', help=stringWithVars("perform one of {sync_commands} on a syncfile, or one of {git_commands} on all the repos in the bank."), choices=commands, default='gitstatus')
    parser.add_argument("--syncfile", metavar="SYNCFILE", help="the path to the syncfile", default='Automatic')
    parser.add_argument("--cwd", metavar="CWD", help="prefix / change the working directory for the repos in the sync file", default="Automatic")
    parser.add_argument("--matching", metavar="MATCH", help=stringWithVars('specify how we can recognize a revision "match": {matchingOpts}'), choices=matchingOpts, default="Automatic")
    parser.add_argument("--verbosity", metavar="NUM", help="Specify the level of feedback detail for the install", default="Automatic")
    parser.add_argument('--dryRun',dest='dryRun',action='store_true', help="Print what would happen instead of executing the deploy")
    parser.add_argument('--version',dest='version',action='store_true', help="Show the version number of the banksync tool")
    parser.set_defaults(dryrun=False)
    parser.set_defaults(version=False)
    argcomplete.autocomplete(parser)

    def printVersionAndExit():
        printWithVars1("banksync {__version__}. Author: Jason F Harris.\nhttps://github.com/jasonfharris/banksync")
        sys.exit(0)

    if len(sys.argv)==1:
        parser.print_help()
        sys.exit(1)

    if len(sys.argv)==2 and sys.argv[1] == '--version':
        printVersionAndExit()

    args, remainingArgs = parser.parse_known_args()
    remainingArgs = [correctlyQuoteArg(arg) for arg in remainingArgs]
    if args.version:
        printVersionAndExit()

    return args, remainingArgs

# --------------------------------------------------------------------------------------------------------------------------
# command "sync"
# --------------------------------------------------------------------------------------------------------------------------

def commandSync():
    checkForSyncRepo(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    allFound = True

    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        repoString = paddedRepoName(repoName,syncDict)
        greenRepoString  = colored(repoString, 'green')
        redRepoString    = colored(repoString, 'red')
        yellowRepoString = colored(repoString, 'yellow')
        if not checkForRepo(repoString, absRepoPath):
            allFound = False
            continue

        found = False
        for method in tryOrder:
            if found:
                break

            if (method == "sha") and ("sha" in repoInfo):
                hash = repoInfo["sha"]
                res = gitCommand("git checkout -B {defaultSyncPointBranchName} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                if res["code"] == 0:
                    revNum = getRevNumber(hash, absRepoPath)
                    shortHash = hash[0:12]
                    printWithVars2("{greenRepoString}: successfully checked out revision by {method}: {shortHash} (revision number {revNum})")
                    found = True
                    break
                printWithVars3("{repoString}: failed to check out revision by {method}: {hash}")

            if (method == "UnixTimeStamp") and ("UnixTimeStamp" in repoInfo):
                if (matching == 'timestamp') or (matching == 'closetimestamp'):
                    ts = repoInfo["UnixTimeStamp"]
                    date = dateFromTimeStamp(ts)
                    res = gitCommand("git log --all --format=format:'\"%at\" : \"%H\",'", 4, cwd=absRepoPath, raiseOnFailure=True, verbosity=verbosity, permitShowingStdOut=False, permitShowingStdErr=False)
                    shaHash = 0
                    if res["code"] == 0:
                        timestampsToShas = json.loads('{'+res["stdout"][0:-1]+'}')
                        if (ts in timestampsToShas):
                            if (matching == 'timestamp') or (matching == 'closetimestamp'):
                                hash = timestampsToShas[ts]
                                branch=defaultSyncPointBranchName
                                res = gitCommand("git checkout -B {branch} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                                if res["code"] == 0:
                                    revNum = getRevNumber(hash, absRepoPath)
                                    printWithVars2("{greenRepoString}: successfully checked out revision by {method}: {ts} ({date}) {hash} (revision number {revNum})")
                                    found = True
                                    break
                        else:
                            if matching == 'closetimestamp':
                                closestTimestamp = min(timestampsToShas, key=lambda x:abs(int(x)-int(ts)))
                                closestDate = dateFromTimeStamp(closestTimestamp)
                                hash = timestampsToShas[closestTimestamp]
                                branch=defaultSyncPointBranchName
                                res = gitCommand("git checkout -B {branch} {hash}", 3, cwd=absRepoPath, verbosity=verbosity)
                                if res["code"] == 0:
                                    revNum = getRevNumber(hash, absRepoPath)
                                    printWithVars2("{yellowRepoString}: warning checking out revision by closest timestamp.", "red")
                                    printWithVars2("       requested {method}: {ts} ({date})")
                                    printWithVars2("       used      {method}: {closestTimestamp} ({closestDate}) {hash} (revision number {revNum})")
                                    found = True
                                    break

                    printWithVars3("{repoString}: failed to check out revision by {method}: {ts} {date}")

        if not found:
            allFound = False
            printWithVars2("{redRepoString}: failed to check out specified revision by any method.")
    if allFound:
        print colored("success! all repos checked out to the specified sync state.", 'green')
    else:
        print colored("failure! not all repos checked out to the specified sync state.", 'red')
        sys.exit(1)

# --------------------------------------------------------------------------------------------------------------------------
# command "createSyncPoint"
# --------------------------------------------------------------------------------------------------------------------------

def commandSyncCreateSyncPoint():
    checkForSyncRepo(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    newSyncDict = OrderedDict(syncDict)
    anyFailures = False
    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        repoString = paddedRepoName(repoName,syncDict)
        greenRepoString = colored(repoString, 'green')
        redRepoString   = colored(repoString, 'red')
        if not checkForRepo(repoString, absRepoPath):
            anyFailures = True
            continue
        (worked, newRepoInfo) = dictFromCurrentRepoState(repoInfo["path"], cwd=cwd, verbosity=verbosity)
        if worked:
            shortHash = newRepoInfo["sha"][0:12]
            date = newRepoInfo["date"]
            printWithVars2("{greenRepoString}: recording bank sync state of {shortHash}, {date}.")
        else:
            printWithVars2("{redRepoString}: failure! not able to get the status of {repoName} at {absRepoPath}", 'red')
            anyFailures = True
        newSyncDict[repoName] = newRepoInfo

    writeDictToSyncFile(syncFilePath, newSyncDict)

    if anyFailures:
        printWithVars1("failure! not all constituent repos had their state recorded.", 'red')
        sys.exit(1)
    else:
        printWithVars1("success! all constituent repos had their state recorded.", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# command "generateSyncFile"
# --------------------------------------------------------------------------------------------------------------------------

def commandGenerateSyncFile():
    checkForSyncRepoDir(syncFilePath, existing = False)
    newSyncDict = OrderedDict()
    anyFailures = False
    for repo in remainingArgs:
        absRepoPath = getAbsRepoPath(repo, cwd)
        repoName = os.path.basename(absRepoPath)
        if not checkForRepo(repoName, absRepoPath):
            anyFailures = True
            continue
        (worked, newRepoInfo) = dictFromCurrentRepoState(repo, cwd=cwd, verbosity=verbosity)
        if worked:
            shortHash = newRepoInfo["sha"][0:12]
            date = newRepoInfo["date"]
            printWithVars2("{repoName}: recording repository state of {shortHash}, {date}.")
        else:
            printWithVars2("failure! not able to get the status of {repoName} at {absRepoPath}", 'red')
            anyFailures = True
        newSyncDict[repoName] = newRepoInfo

    writeDictToSyncFile(syncFilePath, newSyncDict)

    if anyFailures:
        printWithVars1("failure! not all constituent repos had their state recorded.", 'red')
        sys.exit(1)
    else:
        printWithVars1("success! all constituent repos had their state recorded.", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# command "bisect"
# --------------------------------------------------------------------------------------------------------------------------

def commandBisect():
    bisectCmd = "git bisect " + " ".join(remainingArgs)
    res = gitCommand(bisectCmd, 2, cwd=syncRepoPath, verbosity=verbosity);
    commandSync()

# --------------------------------------------------------------------------------------------------------------------------
# a git command
# --------------------------------------------------------------------------------------------------------------------------

def distributeGitCommand(command):
    cmd = command[3:]
    gitCmd = "git " + command[3:] + " " + " ".join(remainingArgs)
    gitCmd = gitCmd.strip()
    checkForSyncRepoDir(syncFilePath)
    syncDict = loadSyncFileAsDict(syncFilePath)
    anyFailures = False
    for repoName in syncDict:
        repoInfo = syncDict[repoName]
        absRepoPath = getAbsRepoPath(repoInfo["path"], cwd)
        if not checkForRepo(repoName, absRepoPath):
            anyFailures = True
            continue
        res = gitCommand(gitCmd, 2, cwd=absRepoPath, verbosity=verbosity);
    if anyFailures:
        printWithVars1("failure! not all constituent repos present.", 'red')
        sys.exit(1)
    else:
        printWithVars1("all constituent repos issued git command '{gitCmd}'", 'green')

# --------------------------------------------------------------------------------------------------------------------------
# dispatch command
# --------------------------------------------------------------------------------------------------------------------------

def dispatchCommand(command):
    #from pudb import set_trace; set_trace()
    if command == "sync":
        commandSync()
    if command == "createSyncPoint":
        commandSyncCreateSyncPoint()
    if command == "generateSyncFile":
        commandGenerateSyncFile()
    if command == "bisect":
        commandBisect()
    if command in git_commands:
        distributeGitCommand(command)

# --------------------------------------------------------------------------------------------------------------------------
# Extract and clean the argument parameters
# --------------------------------------------------------------------------------------------------------------------------

def main():
    global remainingArgs, syncFilePath, syncRepoPath, cwd, matching, verbosity
    args, remainingArgs = parseArguments()

    # Get the config file path
    if args.cwd != "Automatic":
        configFile = os.path.abspath(os.path.join(args.cwd, 'bankconfig.ini'))
    else:
        configFile = os.path.abspath('bankconfig.ini')

    command = args.command
    syncFilePath = getSetting(args.syncfile, configFile, 'syncfile', 'syncfile.wl')
    syncRepoPath = os.path.dirname(os.path.abspath(syncFilePath))
    cwd = getSetting(args.cwd, configFile, 'cwd', '.')
    matching = getSetting(args.matching, configFile, 'matching', 'closetimestamp')
    verbosity= getSetting(args.verbosity, configFile, 'verbosity', 2)
    set_defaults('verbosity', verbosity)
    set_defaults('dryRun', args.dryRun)

    dispatchCommand(command)

if __name__ == '__main__':
    main()<|endoftext|>


<commit_before>from django.core.management.base import NoArgsCommand, CommandError
from django.conf import settings

from jogging import logging

from machiavelli import models

class Command(NoArgsCommand):
	"""
This script checks in every active game if the current turn must change. This happens either
when all the players have finished OR the time limit is exceeded
	"""
	help = 'This script checks in every active game if the current turn must change. \
	This happens either when all the players have finished OR the time limit is exceeded.'

	def handle_noargs(self, **options):
		active_games = models.Game.objects.exclude(phase=0)
		for game in active_games:
			game.check_finished_phase()
<commit_msg>changed check_turns command so that in case of an errror in a game, the rest of the games are also checked<commit_after>from django.core.management.base import NoArgsCommand, CommandError
from django.conf import settings

from jogging import logging

from machiavelli import models

class Command(NoArgsCommand):
	"""
This script checks in every active game if the current turn must change. This happens either
when all the players have finished OR the time limit is exceeded
	"""
	help = 'This script checks in every active game if the current turn must change. \
	This happens either when all the players have finished OR the time limit is exceeded.'

	def handle_noargs(self, **options):
		active_games = models.Game.objects.exclude(phase=0)
		for game in active_games:
			try:
				game.check_finished_phase()
			except Exception, e:
				print "Error while checking if phase is finished in game %s\n\n" % game.pk
				print e
				continue

<|endoftext|>


<commit_before>__version_tuple__ = (0, 4, 0)
__version__ = '0.4.0'
<commit_msg>Release 0.4.1 of vaex-arrow<commit_after>__version_tuple__ = (0, 4, 1)
__version__ = '0.4.1'
<|endoftext|>