<commit_before># -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2017, 2018 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""REANA client utils."""
import base64
import json
import logging
import os
import subprocess
import sys
import traceback
from uuid import UUID

import click
import yadageschemas
import yaml
from jsonschema import ValidationError, validate
from reana_commons.serial import serial_load
from reana_commons.utils import get_workflow_status_change_verb

from reana_client.config import (reana_yaml_schema_file_path,
                                 reana_yaml_valid_file_names)

def workflow_uuid_or_name(ctx, param, value):
    """Get UUID of workflow from configuration / cache file based on name."""
    if not value:
        click.echo(click.style(
            'Workflow name must be provided either with '
            '`--workflow` option or with REANA_WORKON '
            'environment variable', fg='red'),
            err=True)
    else:
        return value

def yadage_load(workflow_file, toplevel='.'):
    """Validate and return yadage workflow specification.

    :param workflow_file: A specification file compliant with
        `yadage` workflow specification.
    :type workflow_file: string
    :param toplevel: URL/path for the workflow file
    :type toplevel: string

    :returns: A dictionary which represents the valid `yadage` workflow.
    """
    schema_name = 'yadage/workflow-schema'
    schemadir = None

    specopts = {
        'toplevel': toplevel,
        'schema_name': schema_name,
        'schemadir': schemadir,
        'load_as_ref': False,
    }

    validopts = {
        'schema_name': schema_name,
        'schemadir': schemadir,
    }

    return yadageschemas.load(spec=workflow_file, specopts=specopts,
                              validopts=validopts, validate=True)

def cwl_load(workflow_file):
    """Validate and return cwl workflow specification.

    :param workflow_file: A specification file compliant with
        `cwl` workflow specification.
    :returns: A dictionary which represents the valid `cwl` workflow.
    """
    result = \
        subprocess.run(
            ['cwltool', '--pack', '--quiet', workflow_file],
            stdout=subprocess.PIPE)
    value = result.stdout.decode('utf-8')
    return json.loads(value)

workflow_load = {
    'yadage': yadage_load,
    'cwl': cwl_load,
    'serial': serial_load,
}
"""Dictionary to extend with new workflow specification loaders."""

def load_workflow_spec(workflow_type, workflow_file, **kwargs):
    """Validate and return machine readable workflow specifications.

    :param workflow_type: A supported workflow specification type.
    :param workflow_file: A workflow file compliant with `workflow_type`
        specification.
    :returns: A dictionary which represents the valid workflow specification.
    """
    return workflow_load[workflow_type](workflow_file, **kwargs)

def load_reana_spec(filepath, skip_validation=False):
    """Load and validate reana specification file.

    :raises IOError: Error while reading REANA spec file from given filepath`.
    :raises ValidationError: Given REANA spec file does not validate against
        REANA specification.
    """
    try:
        with open(filepath) as f:
            reana_yaml = yaml.load(f.read(), Loader=yaml.FullLoader)

        if not skip_validation:
            logging.info('Validating REANA specification file: {filepath}'
                         .format(filepath=filepath))
            _validate_reana_yaml(reana_yaml)

        kwargs = {}
        if reana_yaml['workflow']['type'] == 'serial':
            kwargs['specification'] = reana_yaml['workflow'].\
                get('specification')
            kwargs['parameters'] = \
                reana_yaml.get('inputs', {}).get('parameters', {})
            kwargs['original'] = True

        reana_yaml['workflow']['specification'] = load_workflow_spec(
            reana_yaml['workflow']['type'],
            reana_yaml['workflow'].get('file'),
            **kwargs
        )

        if reana_yaml['workflow']['type'] == 'cwl' and \
                'inputs' in reana_yaml:
            with open(reana_yaml['inputs']['parameters']['input']) as f:
                reana_yaml['inputs']['parameters'] = \
                    yaml.load(f, Loader=yaml.FullLoader)

        return reana_yaml
    except IOError as e:
        logging.info(
            'Something went wrong when reading specifications file from '
            '{filepath} : \n'
            '{error}'.format(filepath=filepath, error=e.strerror))
        raise e
    except Exception as e:
        raise e

def _validate_reana_yaml(reana_yaml):
    """Validate REANA specification file according to jsonschema.

    :param reana_yaml: Dictionary which represents REANA specifications file.
    :raises ValidationError: Given REANA spec file does not validate against
        REANA specification schema.
    """
    try:
        with open(reana_yaml_schema_file_path, 'r') as f:
            reana_yaml_schema = json.loads(f.read())

            validate(reana_yaml, reana_yaml_schema)

    except IOError as e:
        logging.info(
            'Something went wrong when reading REANA validation schema from '
            '{filepath} : \n'
            '{error}'.format(filepath=reana_yaml_schema_file_path,
                             error=e.strerror))
        raise e
    except ValidationError as e:
        logging.info('Invalid REANA specification: {error}'
                     .format(error=e.message))
        raise e

def is_uuid_v4(uuid_or_name):
    """Check if given string is a valid UUIDv4."""
    # Based on https://gist.github.com/ShawnMilo/7777304
    try:
        uuid = UUID(uuid_or_name, version=4)
    except Exception:
        return False

    return uuid.hex == uuid_or_name.replace('-', '')

def get_workflow_name_and_run_number(workflow_name):
    """Return name and run_number of a workflow.

    :param workflow_name: String representing Workflow name.

        Name might be in format 'reana.workflow.123' with arbitrary
        number of dot-delimited substrings, where last substring specifies
        the run number of the workflow this workflow name refers to.

        If name does not contain a valid run number, name without run number
        is returned.
    """
    # Try to split a dot-separated string.
    try:
        name, run_number = workflow_name.split('.', 1)

        try:
            float(run_number)
        except ValueError:
            # `workflow_name` was split, so it is a dot-separated string
            # but it didn't contain a valid `run_number`.
            # Assume that this dot-separated string is the name of
            # the workflow and return just this without a `run_number`.
            return workflow_name, ''

        return name, run_number

    except ValueError:
        # Couldn't split. Probably not a dot-separated string.
        # Return the name given as parameter without a `run_number`.
        return workflow_name, ''

def get_workflow_root():
    """Return the current workflow root directory."""
    reana_yaml = get_reana_yaml_file_path()
    workflow_root = os.getcwd()
    while True:
        file_list = os.listdir(workflow_root)
        parent_dir = os.path.dirname(workflow_root)
        if reana_yaml in file_list:
            break
        else:
            if workflow_root == parent_dir:
                click.echo(click.style(
                    'Not a workflow directory (or any of the parent'
                    ' directories).\nPlease upload from inside'
                    ' the directory containing the reana.yaml '
                    'file of your workflow.', fg='red'))
                sys.exit(1)
            else:
                workflow_root = parent_dir
    workflow_root += '/'
    return workflow_root

def validate_cwl_operational_options(operational_options):
    """Validate cwl operational options."""
    forbidden_args = ['--debug', '--tmpdir-prefix', '--tmp-outdir-prefix'
                      '--default-container', '--outdir']
    for option in operational_options:
        if option in forbidden_args:
            click.echo('Operational option {0} are not allowed. \n'
                       .format(operational_options), err=True)
            sys.exit(1)
    cmd = 'cwltool --version {0}'.format(' '.join(operational_options))
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError as err:
        click.echo('Operational options {0} are not valid. \n'
                   '{1}'.format(operational_options, err), err=True)
        sys.exit(1)

def validate_serial_operational_options(operational_options):
    """Return validated serial operational options."""
    try:
        return dict(p.split('=') for p in operational_options)
    except Exception as err:
        click.echo('Operational options {0} are not valid. \n'
                   '{1}'.format(operational_options, err), err=True)
        sys.exit(1)

def validate_input_parameters(live_parameters, original_parameters):
    """Return validated input parameters."""
    parsed_input_parameters = dict(live_parameters)
    for parameter in parsed_input_parameters.keys():
        if parameter not in original_parameters:
            click.echo(
                click.style('Given parameter - {0}, is not in '
                            'reana.yaml'.format(parameter),
                            fg='red'),
                err=True)
            del live_parameters[parameter]
    return live_parameters

def get_workflow_status_change_msg(workflow, status):
    """Choose verb conjugation depending on status.

    :param workflow: Workflow name whose status changed.
    :param status: String which represents the status the workflow changed to.
    """
    return '{workflow} {verb} {status}'.format(
        workflow=workflow, verb=get_workflow_status_change_verb(status),
        status=status)

def parse_secret_from_literal(literal):
    """Parse a literal string, into a secret dict.

    :param literal: String containg a key and a value. (e.g. 'KEY=VALUE')
    :returns secret: Dictionary in the format suitable for sending
    via http request.
    """
    try:
        key, value = literal.split('=', 1)
        secret = {
            key: {
                'value': base64.b64encode(
                    value.encode('utf-8')
                ).decode('utf-8'),
                'type': 'env'
            }
        }
        return secret

    except ValueError as e:
        logging.debug(traceback.format_exc())
        logging.debug(str(e))
        click.echo(
            click.style(
                'Option "{0}" is invalid: \n'
                'For literal strings use "SECRET_NAME=VALUE" format'
                .format(literal),
                fg='red'),
            err=True)

def parse_secret_from_path(path):
    """Parse a file path into a secret dict.

    :param path: Path of the file containing secret
    :returns secret: Dictionary in the format suitable for sending
     via http request.
    """
    try:
        with open(os.path.expanduser(path), 'rb') as file:
            file_name = os.path.basename(path)
            secret = {
                file_name: {
                    'value': base64.b64encode(
                        file.read()).decode('utf-8'),
                    'type': 'file'
                }
            }
        return secret
    except FileNotFoundError as e:
        logging.debug(traceback.format_exc())
        logging.debug(str(e))
        click.echo(
            click.style(
                'File {0} could not be uploaded: {0} does not exist.'
                .format(path),
                fg='red'),
            err=True)

def get_api_url():
    """Obtain REANA server API URL."""
    return os.environ.get('REANA_SERVER_URL')

def get_reana_yaml_file_path():
    """REANA specification file location."""
    matches = [path for path in reana_yaml_valid_file_names
               if os.path.exists(path)]
    if len(matches) == 0:
        click.echo(
            click.style(
                '==> ERROR: No REANA specification file (reana.yaml) found. '
                'Exiting.', fg='red'))
        sys.exit(1)
    if len(matches) > 1:
        click.echo(
            click.style(
                '==> ERROR: Found {0} REANA specification files ({1}). '
                'Please use only one. Exiting.'.format(len(matches),
                                                       ', '.join(matches)),
                fg='red'))
        sys.exit(1)
    for path in reana_yaml_valid_file_names:
        if os.path.exists(path):
            return path
    # If none of the valid paths exists, fall back to reana.yaml.
    return 'reana.yaml'
<commit_msg>validate: enrich yadage validation output error<commit_after># -*- coding: utf-8 -*-
#
# This file is part of REANA.
# Copyright (C) 2017, 2018 CERN.
#
# REANA is free software; you can redistribute it and/or modify it
# under the terms of the MIT License; see LICENSE file for more details.
"""REANA client utils."""
import base64
import json
import logging
import os
import subprocess
import sys
import traceback
from uuid import UUID

import click
import yadageschemas
import yaml
from jsonschema import ValidationError, validate
from reana_commons.serial import serial_load
from reana_commons.utils import get_workflow_status_change_verb

from reana_client.config import (reana_yaml_schema_file_path,
                                 reana_yaml_valid_file_names)

def workflow_uuid_or_name(ctx, param, value):
    """Get UUID of workflow from configuration / cache file based on name."""
    if not value:
        click.echo(click.style(
            'Workflow name must be provided either with '
            '`--workflow` option or with REANA_WORKON '
            'environment variable', fg='red'),
            err=True)
    else:
        return value

def yadage_load(workflow_file, toplevel='.'):
    """Validate and return yadage workflow specification.

    :param workflow_file: A specification file compliant with
        `yadage` workflow specification.
    :type workflow_file: string
    :param toplevel: URL/path for the workflow file
    :type toplevel: string

    :returns: A dictionary which represents the valid `yadage` workflow.
    """
    schema_name = 'yadage/workflow-schema'
    schemadir = None

    specopts = {
        'toplevel': toplevel,
        'schema_name': schema_name,
        'schemadir': schemadir,
        'load_as_ref': False,
    }

    validopts = {
        'schema_name': schema_name,
        'schemadir': schemadir,
    }

    try:
        return yadageschemas.load(spec=workflow_file, specopts=specopts,
                                  validopts=validopts, validate=True)

    except ValidationError as e:
        e.message = str(e)
        raise e

def cwl_load(workflow_file):
    """Validate and return cwl workflow specification.

    :param workflow_file: A specification file compliant with
        `cwl` workflow specification.
    :returns: A dictionary which represents the valid `cwl` workflow.
    """
    result = \
        subprocess.run(
            ['cwltool', '--pack', '--quiet', workflow_file],
            stdout=subprocess.PIPE)
    value = result.stdout.decode('utf-8')
    return json.loads(value)

workflow_load = {
    'yadage': yadage_load,
    'cwl': cwl_load,
    'serial': serial_load,
}
"""Dictionary to extend with new workflow specification loaders."""

def load_workflow_spec(workflow_type, workflow_file, **kwargs):
    """Validate and return machine readable workflow specifications.

    :param workflow_type: A supported workflow specification type.
    :param workflow_file: A workflow file compliant with `workflow_type`
        specification.
    :returns: A dictionary which represents the valid workflow specification.
    """
    return workflow_load[workflow_type](workflow_file, **kwargs)

def load_reana_spec(filepath, skip_validation=False):
    """Load and validate reana specification file.

    :raises IOError: Error while reading REANA spec file from given filepath`.
    :raises ValidationError: Given REANA spec file does not validate against
        REANA specification.
    """
    try:
        with open(filepath) as f:
            reana_yaml = yaml.load(f.read(), Loader=yaml.FullLoader)

        if not skip_validation:
            logging.info('Validating REANA specification file: {filepath}'
                         .format(filepath=filepath))
            _validate_reana_yaml(reana_yaml)

        kwargs = {}
        if reana_yaml['workflow']['type'] == 'serial':
            kwargs['specification'] = reana_yaml['workflow'].\
                get('specification')
            kwargs['parameters'] = \
                reana_yaml.get('inputs', {}).get('parameters', {})
            kwargs['original'] = True

        reana_yaml['workflow']['specification'] = load_workflow_spec(
            reana_yaml['workflow']['type'],
            reana_yaml['workflow'].get('file'),
            **kwargs
        )

        if reana_yaml['workflow']['type'] == 'cwl' and \
                'inputs' in reana_yaml:
            with open(reana_yaml['inputs']['parameters']['input']) as f:
                reana_yaml['inputs']['parameters'] = \
                    yaml.load(f, Loader=yaml.FullLoader)

        return reana_yaml
    except IOError as e:
        logging.info(
            'Something went wrong when reading specifications file from '
            '{filepath} : \n'
            '{error}'.format(filepath=filepath, error=e.strerror))
        raise e
    except Exception as e:
        raise e

def _validate_reana_yaml(reana_yaml):
    """Validate REANA specification file according to jsonschema.

    :param reana_yaml: Dictionary which represents REANA specifications file.
    :raises ValidationError: Given REANA spec file does not validate against
        REANA specification schema.
    """
    try:
        with open(reana_yaml_schema_file_path, 'r') as f:
            reana_yaml_schema = json.loads(f.read())

            validate(reana_yaml, reana_yaml_schema)

    except IOError as e:
        logging.info(
            'Something went wrong when reading REANA validation schema from '
            '{filepath} : \n'
            '{error}'.format(filepath=reana_yaml_schema_file_path,
                             error=e.strerror))
        raise e
    except ValidationError as e:
        logging.info('Invalid REANA specification: {error}'
                     .format(error=e.message))
        raise e

def is_uuid_v4(uuid_or_name):
    """Check if given string is a valid UUIDv4."""
    # Based on https://gist.github.com/ShawnMilo/7777304
    try:
        uuid = UUID(uuid_or_name, version=4)
    except Exception:
        return False

    return uuid.hex == uuid_or_name.replace('-', '')

def get_workflow_name_and_run_number(workflow_name):
    """Return name and run_number of a workflow.

    :param workflow_name: String representing Workflow name.

        Name might be in format 'reana.workflow.123' with arbitrary
        number of dot-delimited substrings, where last substring specifies
        the run number of the workflow this workflow name refers to.

        If name does not contain a valid run number, name without run number
        is returned.
    """
    # Try to split a dot-separated string.
    try:
        name, run_number = workflow_name.split('.', 1)

        try:
            float(run_number)
        except ValueError:
            # `workflow_name` was split, so it is a dot-separated string
            # but it didn't contain a valid `run_number`.
            # Assume that this dot-separated string is the name of
            # the workflow and return just this without a `run_number`.
            return workflow_name, ''

        return name, run_number

    except ValueError:
        # Couldn't split. Probably not a dot-separated string.
        # Return the name given as parameter without a `run_number`.
        return workflow_name, ''

def get_workflow_root():
    """Return the current workflow root directory."""
    reana_yaml = get_reana_yaml_file_path()
    workflow_root = os.getcwd()
    while True:
        file_list = os.listdir(workflow_root)
        parent_dir = os.path.dirname(workflow_root)
        if reana_yaml in file_list:
            break
        else:
            if workflow_root == parent_dir:
                click.echo(click.style(
                    'Not a workflow directory (or any of the parent'
                    ' directories).\nPlease upload from inside'
                    ' the directory containing the reana.yaml '
                    'file of your workflow.', fg='red'))
                sys.exit(1)
            else:
                workflow_root = parent_dir
    workflow_root += '/'
    return workflow_root

def validate_cwl_operational_options(operational_options):
    """Validate cwl operational options."""
    forbidden_args = ['--debug', '--tmpdir-prefix', '--tmp-outdir-prefix'
                      '--default-container', '--outdir']
    for option in operational_options:
        if option in forbidden_args:
            click.echo('Operational option {0} are not allowed. \n'
                       .format(operational_options), err=True)
            sys.exit(1)
    cmd = 'cwltool --version {0}'.format(' '.join(operational_options))
    try:
        subprocess.check_call(cmd, shell=True)
    except subprocess.CalledProcessError as err:
        click.echo('Operational options {0} are not valid. \n'
                   '{1}'.format(operational_options, err), err=True)
        sys.exit(1)

def validate_serial_operational_options(operational_options):
    """Return validated serial operational options."""
    try:
        return dict(p.split('=') for p in operational_options)
    except Exception as err:
        click.echo('Operational options {0} are not valid. \n'
                   '{1}'.format(operational_options, err), err=True)
        sys.exit(1)

def validate_input_parameters(live_parameters, original_parameters):
    """Return validated input parameters."""
    parsed_input_parameters = dict(live_parameters)
    for parameter in parsed_input_parameters.keys():
        if parameter not in original_parameters:
            click.echo(
                click.style('Given parameter - {0}, is not in '
                            'reana.yaml'.format(parameter),
                            fg='red'),
                err=True)
            del live_parameters[parameter]
    return live_parameters

def get_workflow_status_change_msg(workflow, status):
    """Choose verb conjugation depending on status.

    :param workflow: Workflow name whose status changed.
    :param status: String which represents the status the workflow changed to.
    """
    return '{workflow} {verb} {status}'.format(
        workflow=workflow, verb=get_workflow_status_change_verb(status),
        status=status)

def parse_secret_from_literal(literal):
    """Parse a literal string, into a secret dict.

    :param literal: String containg a key and a value. (e.g. 'KEY=VALUE')
    :returns secret: Dictionary in the format suitable for sending
    via http request.
    """
    try:
        key, value = literal.split('=', 1)
        secret = {
            key: {
                'value': base64.b64encode(
                    value.encode('utf-8')
                ).decode('utf-8'),
                'type': 'env'
            }
        }
        return secret

    except ValueError as e:
        logging.debug(traceback.format_exc())
        logging.debug(str(e))
        click.echo(
            click.style(
                'Option "{0}" is invalid: \n'
                'For literal strings use "SECRET_NAME=VALUE" format'
                .format(literal),
                fg='red'),
            err=True)

def parse_secret_from_path(path):
    """Parse a file path into a secret dict.

    :param path: Path of the file containing secret
    :returns secret: Dictionary in the format suitable for sending
     via http request.
    """
    try:
        with open(os.path.expanduser(path), 'rb') as file:
            file_name = os.path.basename(path)
            secret = {
                file_name: {
                    'value': base64.b64encode(
                        file.read()).decode('utf-8'),
                    'type': 'file'
                }
            }
        return secret
    except FileNotFoundError as e:
        logging.debug(traceback.format_exc())
        logging.debug(str(e))
        click.echo(
            click.style(
                'File {0} could not be uploaded: {0} does not exist.'
                .format(path),
                fg='red'),
            err=True)

def get_api_url():
    """Obtain REANA server API URL."""
    return os.environ.get('REANA_SERVER_URL')

def get_reana_yaml_file_path():
    """REANA specification file location."""
    matches = [path for path in reana_yaml_valid_file_names
               if os.path.exists(path)]
    if len(matches) == 0:
        click.echo(
            click.style(
                '==> ERROR: No REANA specification file (reana.yaml) found. '
                'Exiting.', fg='red'))
        sys.exit(1)
    if len(matches) > 1:
        click.echo(
            click.style(
                '==> ERROR: Found {0} REANA specification files ({1}). '
                'Please use only one. Exiting.'.format(len(matches),
                                                       ', '.join(matches)),
                fg='red'))
        sys.exit(1)
    for path in reana_yaml_valid_file_names:
        if os.path.exists(path):
            return path
    # If none of the valid paths exists, fall back to reana.yaml.
    return 'reana.yaml'
<|endoftext|>


<commit_before>import stat
import typing as tp
import os
import sys
import warnings

def whereis(name: str) -> tp.Iterator[str]:
    """
    Looking in PATH return a sequence of executable files having provided name.

    Additionally, on Windows, it will use PATHEXT.

    .. note:: on Windows name is supposed to be without extension!

    :param name: name of the executable to search for
    :return: an iterator of absolute paths to given executable
    """
    if sys.platform.startswith('win'):
        paths_to_look_in = os.environ.get('PATH', '').split(';')
        name = name.upper()
        available_extensions = os.environ.get('PATHEXT', '.com;.bat;.exe').upper().split(';')
    else:
        paths_to_look_in = os.environ.get('PATH', '').split(':')
        available_extensions = '',

    for directory in paths_to_look_in:
        for file in os.listdir(directory):
            if 'x' not in stat.filemode(os.stat(os.path.join(directory, file)).st_mode):
                continue

            if sys.platform.startswith('win'):  # a POSIX-specific check
                file = file.upper()     # paths are not case-sensitive on Windows

            for extension in available_extensions:
                if file == '%s%s' % (name, extension):
                    yield os.path.join(directory, file)

def is_running_as_root() -> bool:
    """
    Is this process running as root?

    Checks whether effective UID is 0

    :return: bool
    :raises OSError: called on Windows!
    """

    if sys.platform.startswith('win'):
        raise OSError('Routine unavailable on Windows!')

    return os.geteuid() == 0

def suicide(kill_entire_pg: bool = True) -> None:
    """
    Kill self.

    :param kill_entire_pg: whether to kill entire PG if a session leader. Won't work on Windows.
    """
    my_pid = os.getpid()
    if sys.platform.startswith('win'):
        if kill_entire_pg:
            warnings.warn('Windows does not support process leader groups', RuntimeWarning)
        os.kill(my_pid, -9)

    if kill_entire_pg and os.getpgid(0) == my_pid:
        os.killpg(my_pid, 9)
    else:
        os.kill(my_pid, 9)
<commit_msg>refactor<commit_after>import stat
import typing as tp
import os
import sys
import warnings

def whereis(name: str) -> tp.Iterator[str]:
    """
    Looking in PATH return a sequence of executable files having provided name.

    Additionally, on Windows, it will use PATHEXT.

    .. note:: on Windows name is supposed to be without extension!

    :param name: name of the executable to search for
    :return: an iterator of absolute paths to given executable
    """
    if sys.platform.startswith('win'):
        paths_to_look_in = os.environ.get('PATH', '').split(';')
        name = name.upper()
        available_extensions = os.environ.get('PATHEXT', '.com;.bat;.exe').upper().split(';')
    else:
        paths_to_look_in = os.environ.get('PATH', '').split(':')
        available_extensions = '',

    for directory in paths_to_look_in:
        for file in os.listdir(directory):
            path = os.path.join(directory, file)
            if 'x' not in stat.filemode(os.stat(path).st_mode):
                continue

            if sys.platform.startswith('win'):  # a POSIX-specific check
                file = file.upper()     # paths are not case-sensitive on Windows

            for extension in available_extensions:
                if file == '%s%s' % (name, extension):
                    yield path

def is_running_as_root() -> bool:
    """
    Is this process running as root?

    Checks whether effective UID is 0

    :return: bool
    :raises OSError: called on Windows!
    """

    if sys.platform.startswith('win'):
        raise OSError('Routine unavailable on Windows!')

    return os.geteuid() == 0

def suicide(kill_entire_pg: bool = True) -> None:
    """
    Kill self.

    :param kill_entire_pg: whether to kill entire PG if a session leader. Won't work on Windows.
    """
    my_pid = os.getpid()
    if sys.platform.startswith('win'):
        if kill_entire_pg:
            warnings.warn('Windows does not support process leader groups', RuntimeWarning)
        os.kill(my_pid, -9)

    if kill_entire_pg and os.getpgid(0) == my_pid:
        os.killpg(my_pid, 9)
    else:
        os.kill(my_pid, 9)
<|endoftext|>


<commit_before>from copy import copy
import venusian
from .error import ConflictError, DirectiveError, DirectiveReportError
from .framehack import caller_package

class Configurable(object):
    """Object to which configuration actions apply.

    Actions can be added to a configurable. The configurable is then
    prepared. This checks for any conflicts between configurations and
    the configurable is expanded with any configurations from its
    extends list. Then the configurable can be performed, meaning all
    its actions will be performed (to it).
    """
    def __init__(self, extends=None, testing_config=None):
        """
        :param extends:
          the configurables that this configurable extends. Optional.
        :type extends: list of configurables, single configurable.
        :param testing_config:
          We can pass a config object used during testing. This causes
          the actions to be issued against the configurable directly
          instead of waiting for Venusian scanning. This allows
          the use of directive decorators in tests where scanning is
          not an option. Optional, default no testing config.
        """
        if extends is None:
            extends = []
        if not isinstance(extends, list):
            extends = [extends]
        self.extends = extends
        self._testing_config = testing_config
        self.clear()
        if self._testing_config:
            self._testing_config.configurable(self)

    @property
    def testing_config(self):
        return self._testing_config

    @testing_config.setter
    def testing_config(self, config):
        self._testing_config = config
        config.configurable(self)

    def clear(self):
        """Clear any previously registered actions.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`Config.commit`.
        """
        self._actions = []
        self._action_map = None
        self._class_to_actions = None

    def group_actions(self):
        # group actions by class (in fact deepest base class before Directive)
        d = {}
        for action, obj in self._actions:
            d.setdefault(group_key(action), []).append((action, obj))
        # make sure we don't forget about action classes in extends
        for configurable in self.extends:
            for action_class in configurable.action_classes():
                if action_class not in d:
                    d[action_class] = []
        # do the final grouping into Actions objects
        self._class_to_actions = {}
        for action_class, actions in d.items():
            self._class_to_actions[action_class] = Actions(
                actions, self.action_extends(action_class))

    def action_extends(self, action_class):
        return [
            configurable._class_to_actions.get(action_class, Actions([], []))
            for configurable in self.extends]

    def action_classes(self):
        return sort_action_classes(self._class_to_actions.keys())

    def execute(self):
        self.group_actions()
        for action_class in self.action_classes():
            actions = self._class_to_actions.get(action_class)
            if actions is None:
                continue
            actions.prepare(self)
            actions.perform(self)

    def action(self, action, obj):
        """Register an action with configurable.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`Config.commit`.

        :param action: The action to register with the configurable.
        :param obj: The object that this action will be performed on.
        """
        self._actions.append((action, obj))

def group_key(action):
    """We group actions by their deepest base class that's still a real action.

    This way subclasses of for instance the ViewDirective will still
    group with the ViewDirective, so that conflicts can be detected.
    """
    found = None
    for c in action.__class__.__mro__:
        if c is Directive or c is Action:
            return found
        found = c
    assert False

class Actions(object):
    def __init__(self, actions, extends):
        self._actions = actions
        self._action_map = {}
        self.extends = extends

    def prepare(self, configurable):
        """Prepare configurable.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`Config.commit`.

        Detect any conflicts between actions within this
        configurable. Merges in configuration of those configurables
        that this configurable extends.

        Prepare must be called before perform is called.
        """
        # check for conflicts and fill action map
        discriminators = {}
        self._action_map = action_map = {}
        for action, obj in self._actions:
            id = action.identifier(configurable)
            discs = [id]
            discs.extend(action.discriminators())
            for disc in discs:
                other_action = discriminators.get(disc)
                if other_action is not None:
                    raise ConflictError([action, other_action])
                discriminators[disc] = action
            action_map[id] = action, obj
        # inherit from extends
        for extend in self.extends:
            self.combine(extend)

    def combine(self, actions):
        """Combine actions in another prepared configurable with this one.

        Those configuration actions that would conflict are taken to
        have precedence over those in the configurable that is being
        combined with this one. This allows the extending configurable
        to override configuration in extended configurables.

        :param configurable: the configurable to combine with this one.
        """
        to_combine = actions._action_map.copy()
        to_combine.update(self._action_map)
        self._action_map = to_combine

    def perform(self, configurable):
        """Perform actions in this configurable.

        Prepare must be called before calling this.
        """
        values = self._action_map.values()
        values.sort(key=lambda (action, obj): action.order)
        for action, obj in values:
            try:
                action.perform(configurable, obj)
            except DirectiveError, e:
                raise DirectiveReportError(unicode(e), action)

class Action(object):
    """A configuration action.

    A configuration action is performed on an object. Actions can
    conflict with each other based on their identifier and
    discriminators. Actions can override each other based on their
    identifier.

    Can be subclassed to implement concrete configuration actions.

    Action classes can have a ``depends`` attribute, which is a list
    of other action classes that need to be executed before this one
    is. Actions which depend on another will be executed after those
    actions are executed.
    """
    depends = []

    def __init__(self, configurable):
        """Initialize action.

        :param configurable: :class:`morepath.config.Configurable` object
          for which this action was configured.
        """
        self.configurable = configurable
        self.order = None

    def identifier(self, configurable):
        """Returns an immutable that uniquely identifies this config.

        Used for overrides and conflict detection.
        """
        raise NotImplementedError()

    def discriminators(self):
        """Returns a list of immutables to detect conflicts.

        Used for additional configuration conflict detection.
        """
        return []

    def clone(self, **kw):
        """Make a clone of this action.

        Keyword parameters can be used to override attributes in clone.

        Used during preparation to create new fully prepared actions.
        """
        action = copy(self)
        for key, value in kw.items():
            setattr(action, key, value)
        return action

    def prepare(self, obj):
        """Prepare action for configuration.

        :param obj: The object that the action should be performed on.

        Returns an iterable of prepared action, obj tuples.
        """
        return [(self, obj)]

    def perform(self, configurable, obj):
        """Register whatever is being configured with configurable.

        :param configurable: the :class:`morepath.config.Configurable`
          being configured.
        :param obj: the object that the action should be performed on.
        """
        raise NotImplementedError()

class Directive(Action):
    """An :class:`Action` that can be used as a decorator.

    Extends :class:`morepath.config.Action`.

    Base class for concrete Morepath directives such as ``@app.model()``,
    ``@app.view()``, etc.

    Can be used as a Python decorator.

    Can also be used as a context manager for a Python ``with``
    statement. This can be used to provide defaults for the directives
    used within the ``with`` statements context.

    When used as a decorator will track where in the source code
    the directive was used for the purposes of error reporting.
    """

    def __init__(self, configurable):
        """Initialize Directive.

        :param configurable: :class:`morepath.config.Configurable` object
          for which this action was configured.
        """
        super(Directive, self).__init__(configurable)
        self.attach_info = None

    def codeinfo(self):
        """Info about where in the source code the directive was invoked.
        """
        if self.attach_info is None:
            return None
        return self.attach_info.codeinfo

    def __enter__(self):
        return DirectiveAbbreviation(self)

    def __exit__(self, type, value, tb):
        if tb is not None:
            return False

    def __call__(self, wrapped):
        """Call with function to decorate.
        """
        # If we are in testing mode, we immediately add the
        # action.
        if self.configurable._testing_config:
            self.configurable._testing_config.action(self, wrapped)
            return wrapped
        # Normally we only add the action through Venusian scanning.
        def callback(scanner, name, obj):
            scanner.config.action(self, obj)
        self.attach_info = venusian.attach(wrapped, callback)
        return wrapped

class DirectiveAbbreviation(object):
    def __init__(self, directive):
        self.directive = directive

    def __call__(self, **kw):
        return self.directive.clone(**kw)

class Config(object):
    """Contains and executes configuration actions.

    Morepath configuration actions consist of decorator calls on
    :class:`App` instances, i.e. ``@app.view()`` and
    ``@app.model()``. The Config object can scan these configuration
    actions in a package. Once all required configuration is scanned,
    the configuration can be committed. The configuration is then
    processed, associated with :class:`morepath.config.Configurable`
    objects (i.e. :class:`App` objects), conflicts are detected,
    overrides applied, and the configuration becomes final.

    Once the configuration is committed all configured Morepath
    :class:`App` objects are ready to be served using WSGI.

    See :func:`setup`, which creates an instance with standard
    Morepath framework configuration. See also :func:`autoconfig` and
    :func:`autosetup` which help automatically load configuration from
    dependencies.
    """
    def __init__(self):
        self.configurables = []
        self.actions = []
        self.count = 0

    def scan(self, package=None, ignore=None):
        """Scan package for configuration actions (decorators).

        Register any found configuration actions with this
        object. This also includes finding any
        :class:`morepath.config.Configurable` objects.

        :param package: The Python module or package to scan. Optional; if left
          empty case the calling package will be scanned.
        :ignore: A Venusian_ style ignore to ignore some modules during
          scanning. Optional.
        """
        if package is None:
            package = caller_package()
        scanner = venusian.Scanner(config=self)
        scanner.scan(package, ignore=ignore)

    def configurable(self, configurable):
        """Register a configurable with this config.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`scan`.

        A :class:`App` object is a configurable.

        :param: The :class:`morepath.config.Configurable` to register.
        """
        self.configurables.append(configurable)

    def action(self, action, obj):
        """Register an action and obj with this config.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`scan`.

        A Morepath directive decorator is an action, and obj is the
        function that was decorated.

        :param: The :class:`Action` to register.
        :obj: The object to perform action on.
        """
        action.order = self.count
        self.count += 1
        self.actions.append((action, obj))

    def prepared(self):
        """Get prepared actions before they are performed.

        The preparation phase happens as the first stage of a commit.
        This allows configuration actions to complete their
        configuration, do error checking, or transform themselves into
        different configuration actions.

        This calls :meth:`Action.prepare` on all registered configuration
        actions.

        :returns: An iterable of prepared action, obj combinations.
        """
        for action, obj in self.actions:
            for prepared, prepared_obj in action.prepare(obj):
                yield (prepared, prepared_obj)

    def commit(self):
        """Commit all configuration.

        * Clears any previous configuration from all registered
          :class:`morepath.config.Configurable` objects.
        * Prepares actions using :meth:`prepared`.
        * Configuration conflicts within configurables are detected.
        * The configuration of configurable objects that extend
          each other is merged.
        * Finally all configuration actions are performed, completing
          the configuration process.

        This method should be called only once during the lifetime of
        a process, before the configuration is first used. After this
        the configuration is considered to be fixed and cannot be
        further modified. In tests this method can be executed
        multiple times as it will automatically clear the
        configuration of its configurables first.
        """
        # clear all previous configuration; commit can only be run
        # once during runtime so it's handy to clear this out for tests
        for configurable in self.configurables:
            configurable.clear()

        for action, obj in self.prepared():
            action.configurable.action(action, obj)
        configurables = sort_configurables(self.configurables)

        for configurable in configurables:
            configurable.execute()

def topological_sort(l, get_depends):
    result = []
    marked = set()

    def visit(n):
        if n in marked:
            return
        for m in get_depends(n):
            visit(m)
        marked.add(n)
        result.append(n)
    for n in l:
        visit(n)
    return result

def sort_configurables(configurables):
    """Sort configurables topologically by extends.
    """
    return topological_sort(configurables, lambda c: c.extends)

def sort_action_classes(action_classes):
    """Sort action classes topologically by depends.
    """
    return topological_sort(action_classes, lambda c: c.depends)
<commit_msg>A few more cleanups and doc tweaks.<commit_after>from copy import copy
import venusian
from .error import ConflictError, DirectiveError, DirectiveReportError
from .framehack import caller_package

class Configurable(object):
    """Object to which configuration actions apply.

    Actions can be added to a configurable.

    Once all actions are added, the configurable is executed.
    This checks for any conflicts between configurations and
    the configurable is expanded with any configurations from its
    extends list. Then the configurable is performed, meaning all
    its actions will be performed (to it).
    """
    def __init__(self, extends=None, testing_config=None):
        """
        :param extends:
          the configurables that this configurable extends. Optional.
        :type extends: list of configurables, single configurable.
        :param testing_config:
          We can pass a config object used during testing. This causes
          the actions to be issued against the configurable directly
          instead of waiting for Venusian scanning. This allows
          the use of directive decorators in tests where scanning is
          not an option. Optional, default no testing config.
        """
        if extends is None:
            extends = []
        if not isinstance(extends, list):
            extends = [extends]
        self.extends = extends
        self._testing_config = testing_config
        self.clear()
        if self._testing_config:
            self._testing_config.configurable(self)

    @property
    def testing_config(self):
        return self._testing_config

    @testing_config.setter
    def testing_config(self, config):
        self._testing_config = config
        config.configurable(self)

    def clear(self):
        """Clear any previously registered actions.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`Config.commit`.
        """
        self._actions = []
        self._action_map = None
        self._class_to_actions = None

    def group_actions(self):
        """Group actions into :class:`Actions` by class.
        """
        # group actions by class (in fact deepest base class before Directive)
        d = {}
        for action, obj in self._actions:
            d.setdefault(group_key(action), []).append((action, obj))
        # make sure we don't forget about action classes in extends
        for configurable in self.extends:
            for action_class in configurable.action_classes():
                if action_class not in d:
                    d[action_class] = []
        # do the final grouping into Actions objects
        self._class_to_actions = {}
        for action_class, actions in d.items():
            self._class_to_actions[action_class] = Actions(
                actions, self.action_extends(action_class))

    def action_extends(self, action_class):
        """Get actions for action class in extends.
        """
        return [
            configurable._class_to_actions.get(action_class, Actions([], []))
            for configurable in self.extends]

    def action_classes(self):
        """Get action classes sorted in dependency order.
        """
        return sort_action_classes(self._class_to_actions.keys())

    def execute(self):
        """Execute actions for configurable.
        """
        self.group_actions()
        for action_class in self.action_classes():
            actions = self._class_to_actions.get(action_class)
            if actions is None:
                continue
            actions.prepare(self)
            actions.perform(self)

    def action(self, action, obj):
        """Register an action with configurable.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`Config.commit`.

        :param action: The action to register with the configurable.
        :param obj: The object that this action will be performed on.
        """
        self._actions.append((action, obj))

def group_key(action):
    """We group actions by their deepest base class that's still a real action.

    This way subclasses of for instance the ViewDirective will still
    group with the ViewDirective, so that conflicts can be detected.
    """
    found = None
    for c in action.__class__.__mro__:
        if c is Directive or c is Action:
            return found
        found = c
    assert False

class Actions(object):
    def __init__(self, actions, extends):
        self._actions = actions
        self._action_map = {}
        self.extends = extends

    def prepare(self, configurable):
        """Prepare.

        Detect any conflicts between actions.
        Merges in configuration of what this action extends.

        Prepare must be called before perform is called.
        """
        # check for conflicts and fill action map
        discriminators = {}
        self._action_map = action_map = {}
        for action, obj in self._actions:
            id = action.identifier(configurable)
            discs = [id]
            discs.extend(action.discriminators())
            for disc in discs:
                other_action = discriminators.get(disc)
                if other_action is not None:
                    raise ConflictError([action, other_action])
                discriminators[disc] = action
            action_map[id] = action, obj
        # inherit from extends
        for extend in self.extends:
            self.combine(extend)

    def combine(self, actions):
        """Combine another prepared actions with this one.

        Those configuration actions that would conflict are taken to
        have precedence over those being combined with this one. This
        allows the extending actions to override actions in
        extended actions.

        :param actions: the :class:`Actions` to combine with this one.
        """
        to_combine = actions._action_map.copy()
        to_combine.update(self._action_map)
        self._action_map = to_combine

    def perform(self, configurable):
        """Perform actions in this configurable.

        Prepare must be called before calling this.
        """
        values = self._action_map.values()
        values.sort(key=lambda (action, obj): action.order)
        for action, obj in values:
            try:
                action.perform(configurable, obj)
            except DirectiveError, e:
                raise DirectiveReportError(unicode(e), action)

class Action(object):
    """A configuration action.

    A configuration action is performed on an object. Actions can
    conflict with each other based on their identifier and
    discriminators. Actions can override each other based on their
    identifier.

    Can be subclassed to implement concrete configuration actions.

    Action classes can have a ``depends`` attribute, which is a list
    of other action classes that need to be executed before this one
    is. Actions which depend on another will be executed after those
    actions are executed.
    """
    depends = []

    def __init__(self, configurable):
        """Initialize action.

        :param configurable: :class:`morepath.config.Configurable` object
          for which this action was configured.
        """
        self.configurable = configurable
        self.order = None

    def identifier(self, configurable):
        """Returns an immutable that uniquely identifies this config.

        Used for overrides and conflict detection.
        """
        raise NotImplementedError()

    def discriminators(self):
        """Returns a list of immutables to detect conflicts.

        Used for additional configuration conflict detection.
        """
        return []

    def clone(self, **kw):
        """Make a clone of this action.

        Keyword parameters can be used to override attributes in clone.

        Used during preparation to create new fully prepared actions.
        """
        action = copy(self)
        for key, value in kw.items():
            setattr(action, key, value)
        return action

    def prepare(self, obj):
        """Prepare action for configuration.

        :param obj: The object that the action should be performed on.

        Returns an iterable of prepared action, obj tuples.
        """
        return [(self, obj)]

    def perform(self, configurable, obj):
        """Register whatever is being configured with configurable.

        :param configurable: the :class:`morepath.config.Configurable`
          being configured.
        :param obj: the object that the action should be performed on.
        """
        raise NotImplementedError()

class Directive(Action):
    """An :class:`Action` that can be used as a decorator.

    Extends :class:`morepath.config.Action`.

    Base class for concrete Morepath directives such as ``@app.model()``,
    ``@app.view()``, etc.

    Can be used as a Python decorator.

    Can also be used as a context manager for a Python ``with``
    statement. This can be used to provide defaults for the directives
    used within the ``with`` statements context.

    When used as a decorator will track where in the source code
    the directive was used for the purposes of error reporting.
    """

    def __init__(self, configurable):
        """Initialize Directive.

        :param configurable: :class:`morepath.config.Configurable` object
          for which this action was configured.
        """
        super(Directive, self).__init__(configurable)
        self.attach_info = None

    def codeinfo(self):
        """Info about where in the source code the directive was invoked.
        """
        if self.attach_info is None:
            return None
        return self.attach_info.codeinfo

    def __enter__(self):
        return DirectiveAbbreviation(self)

    def __exit__(self, type, value, tb):
        if tb is not None:
            return False

    def __call__(self, wrapped):
        """Call with function to decorate.
        """
        # If we are in testing mode, we immediately add the
        # action.
        if self.configurable._testing_config:
            self.configurable._testing_config.action(self, wrapped)
            return wrapped
        # Normally we only add the action through Venusian scanning.
        def callback(scanner, name, obj):
            scanner.config.action(self, obj)
        self.attach_info = venusian.attach(wrapped, callback)
        return wrapped

class DirectiveAbbreviation(object):
    def __init__(self, directive):
        self.directive = directive

    def __call__(self, **kw):
        return self.directive.clone(**kw)

class Config(object):
    """Contains and executes configuration actions.

    Morepath configuration actions consist of decorator calls on
    :class:`App` instances, i.e. ``@app.view()`` and
    ``@app.model()``. The Config object can scan these configuration
    actions in a package. Once all required configuration is scanned,
    the configuration can be committed. The configuration is then
    processed, associated with :class:`morepath.config.Configurable`
    objects (i.e. :class:`App` objects), conflicts are detected,
    overrides applied, and the configuration becomes final.

    Once the configuration is committed all configured Morepath
    :class:`App` objects are ready to be served using WSGI.

    See :func:`setup`, which creates an instance with standard
    Morepath framework configuration. See also :func:`autoconfig` and
    :func:`autosetup` which help automatically load configuration from
    dependencies.
    """
    def __init__(self):
        self.configurables = []
        self.actions = []
        self.count = 0

    def scan(self, package=None, ignore=None):
        """Scan package for configuration actions (decorators).

        Register any found configuration actions with this
        object. This also includes finding any
        :class:`morepath.config.Configurable` objects.

        :param package: The Python module or package to scan. Optional; if left
          empty case the calling package will be scanned.
        :ignore: A Venusian_ style ignore to ignore some modules during
          scanning. Optional.
        """
        if package is None:
            package = caller_package()
        scanner = venusian.Scanner(config=self)
        scanner.scan(package, ignore=ignore)

    def configurable(self, configurable):
        """Register a configurable with this config.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`scan`.

        A :class:`App` object is a configurable.

        :param: The :class:`morepath.config.Configurable` to register.
        """
        self.configurables.append(configurable)

    def action(self, action, obj):
        """Register an action and obj with this config.

        This is normally not invoked directly, instead is called
        indirectly by :meth:`scan`.

        A Morepath directive decorator is an action, and obj is the
        function that was decorated.

        :param: The :class:`Action` to register.
        :obj: The object to perform action on.
        """
        action.order = self.count
        self.count += 1
        self.actions.append((action, obj))

    def prepared(self):
        """Get prepared actions before they are performed.

        The preparation phase happens as the first stage of a commit.
        This allows configuration actions to complete their
        configuration, do error checking, or transform themselves into
        different configuration actions.

        This calls :meth:`Action.prepare` on all registered configuration
        actions.

        :returns: An iterable of prepared action, obj combinations.
        """
        for action, obj in self.actions:
            for prepared, prepared_obj in action.prepare(obj):
                yield (prepared, prepared_obj)

    def commit(self):
        """Commit all configuration.

        * Clears any previous configuration from all registered
          :class:`morepath.config.Configurable` objects.
        * Prepares actions using :meth:`prepared`.
        * Configuration conflicts within configurables are detected.
        * The configuration of configurable objects that extend
          each other is merged.
        * Finally all configuration actions are performed, completing
          the configuration process.

        This method should be called only once during the lifetime of
        a process, before the configuration is first used. After this
        the configuration is considered to be fixed and cannot be
        further modified. In tests this method can be executed
        multiple times as it will automatically clear the
        configuration of its configurables first.
        """
        # clear all previous configuration; commit can only be run
        # once during runtime so it's handy to clear this out for tests
        for configurable in self.configurables:
            configurable.clear()

        for action, obj in self.prepared():
            action.configurable.action(action, obj)
        configurables = sort_configurables(self.configurables)

        for configurable in configurables:
            configurable.execute()

def topological_sort(l, get_depends):
    result = []
    marked = set()

    def visit(n):
        if n in marked:
            return
        for m in get_depends(n):
            visit(m)
        marked.add(n)
        result.append(n)
    for n in l:
        visit(n)
    return result

def sort_configurables(configurables):
    """Sort configurables topologically by extends.
    """
    return topological_sort(configurables, lambda c: c.extends)

def sort_action_classes(action_classes):
    """Sort action classes topologically by depends.
    """
    return topological_sort(action_classes, lambda c: c.depends)
<|endoftext|>


<commit_before>import os
import pdb
from .precisecodevillemerge import Weave
from .config import FragmentsConfig

def apply(file_name):
    """Revert changes to fragments repository"""
    config = FragmentsConfig()
    weave = Weave()
    changed_path = os.path.realpath(file_name)
    changed_key = changed_path[len(config.root)+1:]
    if changed_key not in config['files']:
        yield "Could not apply changes in %r, it is not being followed" % changed_key
        return
    elif not os.access(changed_path, os.R_OK|os.W_OK):
        yield "Could not apply changes in %r, it no longer exists on disk" % changed_key
        return

    changed_uuid = config['files'][changed_key]
    old_path = os.path.join(config.directory, changed_uuid)

    if not os.access(old_path, os.R_OK|os.W_OK):
        yield "Could not apply changes in %r, it has never been committed" % changed_key
        return

    weave.add_revision(1, file(old_path, 'r').readlines(), [])
    weave.add_revision(2, file(changed_path, 'r').readlines(), [1])

    for i, other_key in enumerate(config['files']):
        other_path = os.path.join(config.root, other_key)
        if other_path == changed_path:
            continue # don't try to apply changes to ourself
        revision = i + 3
        weave.add_revision(revision, file(other_path, 'r').readlines(), [])
        merge_result = weave.cherry_pick(2, revision) # Can I apply changes in revision 2 onto this other file?
        if tuple in (type(mr) for mr in merge_result):
            if len(merge_result) == 1:
                # total conflict, skip
                yield "Changes in %r cannot apply to %r, skipping" % (changed_key, other_key)
                continue
            other_file = file(other_path, 'w')
            for line_or_conflict in merge_result:
                if isinstance(line_or_conflict, basestring):
                    other_file.write(line_or_conflict)
                else:
                    other_file.write('>'*7 + '\n')
                    for line in line_or_conflict[0]:
                        other_file.write(line)
                    other_file.write('='*7 + '\n')
                    for line in line_or_conflict[1]:
                        other_file.write(line)
                    other_file.write('>'*7 + '\n')
            other_file.close()
            yield "Conflict merging %r => %r" % (changed_key, other_key)
        else:
            # Merge is clean:
            other_file = file(other_path, 'w')
            other_file.writelines(merge_result)
            other_file.close()
            yield "Changes in %r applied cleanly to %r" % (changed_key, other_key)
<commit_msg>store third revision for interactive chunk choosing<commit_after>import os
import pdb
from .precisecodevillemerge import Weave
from .config import FragmentsConfig

def apply(file_name):
    """Revert changes to fragments repository"""
    config = FragmentsConfig()
    weave = Weave()
    changed_path = os.path.realpath(file_name)
    changed_key = changed_path[len(config.root)+1:]
    if changed_key not in config['files']:
        yield "Could not apply changes in %r, it is not being followed" % changed_key
        return
    elif not os.access(changed_path, os.R_OK|os.W_OK):
        yield "Could not apply changes in %r, it no longer exists on disk" % changed_key
        return

    changed_uuid = config['files'][changed_key]
    old_path = os.path.join(config.directory, changed_uuid)

    if not os.access(old_path, os.R_OK|os.W_OK):
        yield "Could not apply changes in %r, it has never been committed" % changed_key
        return

    old_revision = 1
    weave.add_revision(old_revision, file(old_path, 'r').readlines(), [])
    new_revision = 2
    weave.add_revision(new_revision, file(changed_path, 'r').readlines(), [])

    changes_to_apply = []
    for i, line_or_tuple in enumerate(weave.merge(old_revision, new_revision)):
        if isinstance(line_or_tuple, tuple):
            for line in line_or_tuple[1]:
                changes_to_apply.append(line)
        else:
            changes_to_apply.append(line_or_tuple)

    changed_revision = 3
    weave.add_revision(changed_revision, changes_to_apply, [1])

    current_revision = changed_revision
    for other_key in config['files']:
        other_path = os.path.join(config.root, other_key)
        if other_path == changed_path:
            continue # don't try to apply changes to ourself
        current_revision += 1
        weave.add_revision(current_revision, file(other_path, 'r').readlines(), [])
        merge_result = weave.cherry_pick(changed_revision, current_revision) # Can I apply changes in changed_revision onto this other file?
        if tuple in (type(mr) for mr in merge_result):
            if len(merge_result) == 1:
                # total conflict, skip
                yield "Changes in %r cannot apply to %r, skipping" % (changed_key, other_key)
                continue
            other_file = file(other_path, 'w')
            for line_or_conflict in merge_result:
                if isinstance(line_or_conflict, basestring):
                    other_file.write(line_or_conflict)
                else:
                    other_file.write('>'*7 + '\n')
                    for line in line_or_conflict[0]:
                        other_file.write(line)
                    other_file.write('='*7 + '\n')
                    for line in line_or_conflict[1]:
                        other_file.write(line)
                    other_file.write('>'*7 + '\n')
            other_file.close()
            yield "Conflict merging %r => %r" % (changed_key, other_key)
        else:
            # Merge is clean:
            other_file = file(other_path, 'w')
            other_file.writelines(merge_result)
            other_file.close()
            yield "Changes in %r applied cleanly to %r" % (changed_key, other_key)
<|endoftext|>


<commit_before># coding: utf-8
from fabric.api import run, env, cd, prefix, shell_env
from config import load_config

config = load_config()
host_string = config.HOST_STRING

def init():
    """初始化部署"""
    env.host_string = config.HOST_STRING
    with cd('/var/www/shufa'):
        with shell_env(MODE='PRODUCTION'):
            run('git reset --hard HEAD')
            run('git pull')
            run('bower install --allow-root')
            with prefix('source venv/bin/activate'):
                run('pip install -r requirements.txt')
                run('python manage.py createdb')
            run('supervisorctl restart shufa')

def deploy():
    """部署"""
    env.host_string = config.HOST_STRING
    with cd('/var/www/shufa'):
        with shell_env(MODE='PRODUCTION'):
            run('git reset --hard HEAD')
            run('git pull')
            run('bower install --allow-root')
            with prefix('source venv/bin/activate'):
                run('pip install -r requirements.txt')
                run('python manage.py db upgrade')
            run('sudo supervisorctl restart shufa')

def restart():
    """重启"""
    env.host_string = config.HOST_STRING
    run('sudo supervisorctl restart shufa')<commit_msg>Update fabfile.<commit_after># coding: utf-8
from fabric.api import run, env, cd, prefix, shell_env
from config import load_config

config = load_config()
host_string = config.HOST_STRING

def init():
    """初始化部署"""
    env.host_string = config.HOST_STRING
    with cd('/var/www/shufa'):
        with shell_env(MODE='PRODUCTION'):
            run('git reset --hard HEAD')
            run('git pull')
            run('bower install --allow-root')
            with prefix('source venv/bin/activate'):
                run('pip install -r requirements.txt')
                run('python manage.py createdb')
            run('supervisorctl restart shufa')

def deploy():
    """部署"""
    env.host_string = config.HOST_STRING
    with cd('/var/www/shufa'):
        with shell_env(MODE='PRODUCTION'):
            run('git reset --hard HEAD')
            run('git pull')
            run('bower install --allow-root')
            with prefix('source venv/bin/activate'):
                run('pip install -r requirements.txt')
                run('python manage.py db upgrade')
            run('supervisorctl restart shufa')

def restart():
    """重启"""
    env.host_string = config.HOST_STRING
    run('sudo supervisorctl restart shufa')<|endoftext|>


<commit_before>#!/usr/bin/env python

"""
Deployment file to facilitate releases of pymatgen.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Apr 29, 2012"

import glob
import os

from fabric.api import local, lcd
from pymatgen import __version__ as ver

def makedoc():
    with lcd("examples"):
        local("ipython nbconvert --to html *.ipynb")
        local("mv *.html ../docs/_static")
    with lcd("docs"):
        local("sphinx-apidoc -o . -f ../pymatgen")
        local("rm pymatgen.*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/pymatgen') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        local("make html")
        local("cp _static/* ../../pymatgen-docs/html/static")

def publish():
    local("python setup.py release")

def test():
    local("nosetests")

def setver():
    local("sed s/version=.*,/version=\\\"{}\\\",/ setup.py > newsetup".format(ver))
    local("mv newsetup setup.py")

def update_dev_doc():
    makedoc()
    with lcd("../pymatgen-docs/html/"):
        local("git add .")
        local("git commit -a -m \"Update dev docs\"")
        local("git push origin gh-pages")

def log_ver():
    filepath = os.path.join(os.environ["HOME"], "Dropbox", "Public", "pymatgen", ver)
    with open(filepath, "w") as f:
        f.write("Release")

def release():
    setver()
    #test()
    publish()
    update_dev_doc()
    log_ver()
<commit_msg>Fix fabfile.<commit_after>#!/usr/bin/env python

"""
Deployment file to facilitate releases of pymatgen.
"""

from __future__ import division

__author__ = "Shyue Ping Ong"
__copyright__ = "Copyright 2012, The Materials Project"
__version__ = "0.1"
__maintainer__ = "Shyue Ping Ong"
__email__ = "shyue@mit.edu"
__date__ = "Apr 29, 2012"

import glob
import os

from fabric.api import local, lcd
from pymatgen import __version__ as ver

def makedoc():
    with lcd("examples"):
        local("ipython nbconvert --to html *.ipynb")
        local("mv *.html ../docs/_static")
    with lcd("docs"):
        local("sphinx-apidoc -o . -f ../pymatgen")
        local("rm pymatgen.*.tests.rst")
        for f in glob.glob("docs/*.rst"):
            if f.startswith('docs/pymatgen') and f.endswith('rst'):
                newoutput = []
                suboutput = []
                subpackage = False
                with open(f, 'r') as fid:
                    for line in fid:
                        clean = line.strip()
                        if clean == "Subpackages":
                            subpackage = True
                        if not subpackage and not clean.endswith("tests"):
                            newoutput.append(line)
                        else:
                            if not clean.endswith("tests"):
                                suboutput.append(line)
                            if clean.startswith("pymatgen") and not clean.endswith("tests"):
                                newoutput.extend(suboutput)
                                subpackage = False
                                suboutput = []

                with open(f, 'w') as fid:
                    fid.write("".join(newoutput))
        local("make html")
        local("cp _static/* ../../pymatgen-docs/html/static")

def publish():
    local("python setup.py release")

def test():
    local("nosetests")

def setver():
    local("sed s/version=.*,/version=\\\"{}\\\",/ setup.py > newsetup"
          .format(ver))
    local("mv newsetup setup.py")

def update_dev_doc():
    makedoc()
    with lcd("../pymatgen-docs/html/"):
        local("git add .")
        local("git commit -a -m \"Update dev docs\"")
        local("git push origin gh-pages")

def log_ver():
    filepath = os.path.join(os.environ["HOME"], "Dropbox", "Public",
                            "pymatgen", ver)
    with open(filepath, "w") as f:
        f.write("Release")

def release():
    setver()
    test()
    publish()
    log_ver()
    update_dev_doc()
<|endoftext|>


<commit_before>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2017 Judit Acs <judit@sch.bme.hu>
#
# Distributed under terms of the MIT license.
from __future__ import unicode_literals

from datetime import datetime
import numpy as np
import logging
import os

import tensorflow as tf

class SimpleSeq2seq(object):
    """Wrapper class for TF legacy seq2seq module"""

    def __init__(self, cell_type, cell_size, embedding_size,
                 layers=None, model_dir=None, **kwargs):
        self.init_cell(cell_type, cell_size, layers)
        self.embedding_size = embedding_size
        self.model_dir = model_dir
        self.result = {}
        tf.reset_default_graph()

    def init_cell(self, cell_type, cell_size, layers=None):
        if layers is None or layers == 1:
            if cell_type == 'LSTM':
                self.cell = lambda: tf.contrib.rnn.BasicLSTMCell(cell_size)
            elif cell_type == 'GRU':
                self.cell = lambda: tf.contrib.rnn.GRUCell(cell_size)
        else:
            if cell_type == 'LSTM':
                self.cell = lambda: tf.contrib.rnn.MultiRNNCell(
                    [tf.contrib.rnn.BasicLSTMCell(cell_size)
                     for _ in range(layers)]
                )
            elif cell_type == 'GRU':
                self.cell = lambda: tf.contrib.rnn.MultiRNNCell(
                    [tf.contrib.rnn.GRUCell(cell_size) for _ in range(layers)]
                )

    def create_model(self, dataset):
        self.create_placeholders(dataset)
        self.create_train_ops(dataset)

    def create_placeholders(self, dataset):
        self.buckets = [(dataset.maxlen_enc, dataset.maxlen_dec-1)]
        self.enc_inp = [
            tf.placeholder(tf.int32, shape=[None], name='enc{}'.format(i))
            for i in range(dataset.maxlen_enc)
        ]
        self.dec_inp = [
            tf.placeholder(tf.int32, shape=[None], name='dec{}'.format(i))
            for i in range(dataset.maxlen_dec)
        ]
        self.target_weights = [
            tf.placeholder(tf.float32, shape=[None], name='target{}'.format(i))
            for i in range(dataset.maxlen_dec)
        ]
        self.feed_previous = tf.placeholder(tf.bool)

    def initialize_seq2seq(self, dataset):
        do, dm = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(
            self.enc_inp, self.dec_inp,
            cell=self.cell(),
            num_encoder_symbols=len(dataset.vocab_enc),
            num_decoder_symbols=len(dataset.vocab_dec),
            embedding_size=self.embedding_size,
            dtype=tf.float32,
            feed_previous=self.feed_previous,
        )
        self.dec_out = do
        self.dec_memory = dm

    def create_train_ops(self, dataset):
        def seq2seq_f(enc_inp, dec_inp, do_decode):
            return tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(
                enc_inp, dec_inp,
                cell=self.cell(),
                num_encoder_symbols=len(dataset.vocab_enc),
                num_decoder_symbols=len(dataset.vocab_dec),
                embedding_size=self.embedding_size,
                dtype=tf.float32,
                feed_previous=do_decode,
            )
        targets = [self.dec_inp[i + 1] for i in range(len(self.dec_inp) - 1)]
        self.outputs, self.loss = tf.contrib.legacy_seq2seq.model_with_buckets(
            self.enc_inp, self.dec_inp,
            targets, self.target_weights, self.buckets,
            lambda x, y: seq2seq_f(x, y, False)
        )

        def create_optimizer():
            return tf.train.RMSPropOptimizer(0.01)

        self.train_ops = [create_optimizer().minimize(l) for l in self.loss]

    def train_and_test(self, dataset, batch_size, epochs=100000,
                       patience=5, val_loss_th=1e-4):
        self.result['train_loss'] = []
        self.result['test_loss'] = []
        self.result['val_loss'] = []
        self.prev_val_loss = -10
        self.result['patience'] = patience
        self.result['val_loss_th'] = val_loss_th
        self.early_cnt = patience
        saver = tf.train.Saver()
        with tf.Session() as sess:
            start = datetime.now()
            sess.run(tf.global_variables_initializer())
            for iter_no in range(epochs):
                self.run_train_step(sess, dataset, batch_size)
                self.run_validation(sess, dataset, iter_no)
                if self.do_early_stopping():
                    logging.info('Early stopping at iteration {}, '
                                 'valid loss: {}'.format(
                                     iter_no+1, self.result['val_loss'][-1]))
                    break
            else:
                logging.info('Training completed without early stopping. '
                             'Iterations run: {}'.format(epochs))
            if self.model_dir is not None:
                model_prefix = os.path.join(self.model_dir, 'model')
                saver.save(sess, model_prefix)
                dataset.save_vocabularies(self.model_dir)
            self.run_test(sess, dataset)
            self.result['epochs_run'] = iter_no+1
            self.result['running_time'] = (datetime.now() -
                                           start).total_seconds()
            # self.run_train_as_test(sess, dataset)

    def do_early_stopping(self):
        if len(self.result['val_loss']) < self.result['patience']:
            return False
        do_stop = False
        try:
            if abs(self.result['val_loss'][-1] - self.result['val_loss'][-2]) \
                    < self.result['val_loss_th']:
                self.early_cnt -= 1
                if self.early_cnt == 0:
                    do_stop = True
            else:
                self.early_cnt = self.result['patience']
        except IndexError:
            pass
        return do_stop

    def run_train_step(self, sess, dataset, batch_size):
        batch_enc, batch_dec = dataset.get_batch(batch_size)
        feed_dict = self.populate_feed_dict(batch_enc, batch_dec)
        feed_dict[self.feed_previous] = True
        _, train_loss = sess.run([self.train_ops, self.loss],
                                 feed_dict=feed_dict)
        self.result['train_loss'].append(sum(train_loss))

    def run_validation(self, sess, dataset, iter_no=None):
        feed_dict = self.populate_feed_dict(dataset.data_enc_valid,
                                            dataset.data_dec_valid)
        feed_dict[self.feed_previous] = True
        val_loss = sess.run(self.loss, feed_dict=feed_dict)
        if iter_no is not None and iter_no % 1000 == 999:
            logging.info('Iter {}, validation loss: {}'.format(
                iter_no+1, val_loss))
        self.result['val_loss'].append(sum(val_loss))

    def run_test(self, sess, dataset, save_output_fn=None):
        test_enc = dataset.data_enc_test
        test_dec = dataset.data_dec_test
        feed_dict = self.populate_feed_dict(test_enc, test_dec)
        feed_dict[self.feed_previous] = True
        test_out, test_loss = sess.run([self.outputs, self.loss],
                                       feed_dict=feed_dict)
        self.result['test_loss'].append(sum(test_loss))

        self.test_out = test_out
        self.dataset = dataset

    def run_train_as_test(self, sess, dataset, batch_size=1024):
        train_enc, train_dec = dataset.get_batch(batch_size)
        feed_dict = self.populate_feed_dict(train_enc, train_dec)
        feed_dict[self.feed_previous] = True
        train_out, train_loss = sess.run([self.outputs, self.loss],
                                         feed_dict=feed_dict)
        self.train_out = train_out

    def save_test_output(self, stream,
                         include_test_input=False):
        self.decode_test()
        test_samples = self.dataset.get_test_samples(
            include_test_input
        )
        if include_test_input:
            test_samples = ['\t'.join(t) for t in test_samples]
        try:
            stream.write('\n'.join('{0}\t{1}'.format(gold, output)
                                   for gold, output in zip(
                                    test_samples, self.decoded)).encode('utf8'))
        except AttributeError:
            stream.write('\n'.join('{0}\t{1}'.format(gold, output)
                                   for gold, output in zip(
                                    test_samples, self.decoded)))

    def decode_test(self, replace_pad=True):
        inv_vocab = {v: k for k, v in self.dataset.vocab_dec.items()}
        indices = np.zeros((self.test_out[0][0].shape[0],
                            len(self.test_out[0])))
        for si in range(len(self.test_out[0])):
            maxi = self.test_out[0][si].argmax(axis=1)
            indices[:, si] = maxi
        decoded = []
        for row in indices:
            dec = ''.join(inv_vocab[r] for r in row)
            if replace_pad:
                dec = dec.replace('PAD', ' ').strip()
                dec = dec.replace('STOP', ' ').strip()
            decoded.append(dec)
        self.decoded = decoded

    def populate_feed_dict(self, batch_enc, batch_dec):
        feed_dict = {}
        batch_size = batch_enc.shape[0]
        target_weights = np.ones((batch_size, batch_dec.shape[1]),
                                 dtype=np.float32)
        for i in range(batch_enc.shape[1]):
            feed_dict[self.enc_inp[i]] = batch_enc[:, i]
        for i in range(batch_dec.shape[1]):
            feed_dict[self.dec_inp[i]] = batch_dec[:, i]
            feed_dict[self.target_weights[i]] = target_weights[:, i]
        return feed_dict
<commit_msg>change exception type<commit_after>#! /usr/bin/env python
# -*- coding: utf-8 -*-
# vim:fenc=utf-8
#
# Copyright © 2017 Judit Acs <judit@sch.bme.hu>
#
# Distributed under terms of the MIT license.
from __future__ import unicode_literals

from datetime import datetime
import numpy as np
import logging
import os

import tensorflow as tf

class SimpleSeq2seq(object):
    """Wrapper class for TF legacy seq2seq module"""

    def __init__(self, cell_type, cell_size, embedding_size,
                 layers=None, model_dir=None, **kwargs):
        self.init_cell(cell_type, cell_size, layers)
        self.embedding_size = embedding_size
        self.model_dir = model_dir
        self.result = {}
        tf.reset_default_graph()

    def init_cell(self, cell_type, cell_size, layers=None):
        if layers is None or layers == 1:
            if cell_type == 'LSTM':
                self.cell = lambda: tf.contrib.rnn.BasicLSTMCell(cell_size)
            elif cell_type == 'GRU':
                self.cell = lambda: tf.contrib.rnn.GRUCell(cell_size)
        else:
            if cell_type == 'LSTM':
                self.cell = lambda: tf.contrib.rnn.MultiRNNCell(
                    [tf.contrib.rnn.BasicLSTMCell(cell_size)
                     for _ in range(layers)]
                )
            elif cell_type == 'GRU':
                self.cell = lambda: tf.contrib.rnn.MultiRNNCell(
                    [tf.contrib.rnn.GRUCell(cell_size) for _ in range(layers)]
                )

    def create_model(self, dataset):
        self.create_placeholders(dataset)
        self.create_train_ops(dataset)

    def create_placeholders(self, dataset):
        self.buckets = [(dataset.maxlen_enc, dataset.maxlen_dec-1)]
        self.enc_inp = [
            tf.placeholder(tf.int32, shape=[None], name='enc{}'.format(i))
            for i in range(dataset.maxlen_enc)
        ]
        self.dec_inp = [
            tf.placeholder(tf.int32, shape=[None], name='dec{}'.format(i))
            for i in range(dataset.maxlen_dec)
        ]
        self.target_weights = [
            tf.placeholder(tf.float32, shape=[None], name='target{}'.format(i))
            for i in range(dataset.maxlen_dec)
        ]
        self.feed_previous = tf.placeholder(tf.bool)

    def initialize_seq2seq(self, dataset):
        do, dm = tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(
            self.enc_inp, self.dec_inp,
            cell=self.cell(),
            num_encoder_symbols=len(dataset.vocab_enc),
            num_decoder_symbols=len(dataset.vocab_dec),
            embedding_size=self.embedding_size,
            dtype=tf.float32,
            feed_previous=self.feed_previous,
        )
        self.dec_out = do
        self.dec_memory = dm

    def create_train_ops(self, dataset):
        def seq2seq_f(enc_inp, dec_inp, do_decode):
            return tf.contrib.legacy_seq2seq.embedding_attention_seq2seq(
                enc_inp, dec_inp,
                cell=self.cell(),
                num_encoder_symbols=len(dataset.vocab_enc),
                num_decoder_symbols=len(dataset.vocab_dec),
                embedding_size=self.embedding_size,
                dtype=tf.float32,
                feed_previous=do_decode,
            )
        targets = [self.dec_inp[i + 1] for i in range(len(self.dec_inp) - 1)]
        self.outputs, self.loss = tf.contrib.legacy_seq2seq.model_with_buckets(
            self.enc_inp, self.dec_inp,
            targets, self.target_weights, self.buckets,
            lambda x, y: seq2seq_f(x, y, False)
        )

        def create_optimizer():
            return tf.train.RMSPropOptimizer(0.01)

        self.train_ops = [create_optimizer().minimize(l) for l in self.loss]

    def train_and_test(self, dataset, batch_size, epochs=100000,
                       patience=5, val_loss_th=1e-4):
        self.result['train_loss'] = []
        self.result['test_loss'] = []
        self.result['val_loss'] = []
        self.prev_val_loss = -10
        self.result['patience'] = patience
        self.result['val_loss_th'] = val_loss_th
        self.early_cnt = patience
        saver = tf.train.Saver()
        with tf.Session() as sess:
            start = datetime.now()
            sess.run(tf.global_variables_initializer())
            for iter_no in range(epochs):
                self.run_train_step(sess, dataset, batch_size)
                self.run_validation(sess, dataset, iter_no)
                if self.do_early_stopping():
                    logging.info('Early stopping at iteration {}, '
                                 'valid loss: {}'.format(
                                     iter_no+1, self.result['val_loss'][-1]))
                    break
            else:
                logging.info('Training completed without early stopping. '
                             'Iterations run: {}'.format(epochs))
            if self.model_dir is not None:
                model_prefix = os.path.join(self.model_dir, 'model')
                saver.save(sess, model_prefix)
                dataset.save_vocabularies(self.model_dir)
            self.run_test(sess, dataset)
            self.result['epochs_run'] = iter_no+1
            self.result['running_time'] = (datetime.now() -
                                           start).total_seconds()
            # self.run_train_as_test(sess, dataset)

    def do_early_stopping(self):
        if len(self.result['val_loss']) < self.result['patience']:
            return False
        do_stop = False
        try:
            if abs(self.result['val_loss'][-1] - self.result['val_loss'][-2]) \
                    < self.result['val_loss_th']:
                self.early_cnt -= 1
                if self.early_cnt == 0:
                    do_stop = True
            else:
                self.early_cnt = self.result['patience']
        except IndexError:
            pass
        return do_stop

    def run_train_step(self, sess, dataset, batch_size):
        batch_enc, batch_dec = dataset.get_batch(batch_size)
        feed_dict = self.populate_feed_dict(batch_enc, batch_dec)
        feed_dict[self.feed_previous] = True
        _, train_loss = sess.run([self.train_ops, self.loss],
                                 feed_dict=feed_dict)
        self.result['train_loss'].append(sum(train_loss))

    def run_validation(self, sess, dataset, iter_no=None):
        feed_dict = self.populate_feed_dict(dataset.data_enc_valid,
                                            dataset.data_dec_valid)
        feed_dict[self.feed_previous] = True
        val_loss = sess.run(self.loss, feed_dict=feed_dict)
        if iter_no is not None and iter_no % 1000 == 999:
            logging.info('Iter {}, validation loss: {}'.format(
                iter_no+1, val_loss))
        self.result['val_loss'].append(sum(val_loss))

    def run_test(self, sess, dataset, save_output_fn=None):
        test_enc = dataset.data_enc_test
        test_dec = dataset.data_dec_test
        feed_dict = self.populate_feed_dict(test_enc, test_dec)
        feed_dict[self.feed_previous] = True
        test_out, test_loss = sess.run([self.outputs, self.loss],
                                       feed_dict=feed_dict)
        self.result['test_loss'].append(sum(test_loss))

        self.test_out = test_out
        self.dataset = dataset

    def run_train_as_test(self, sess, dataset, batch_size=1024):
        train_enc, train_dec = dataset.get_batch(batch_size)
        feed_dict = self.populate_feed_dict(train_enc, train_dec)
        feed_dict[self.feed_previous] = True
        train_out, train_loss = sess.run([self.outputs, self.loss],
                                         feed_dict=feed_dict)
        self.train_out = train_out

    def save_test_output(self, stream,
                         include_test_input=False):
        self.decode_test()
        test_samples = self.dataset.get_test_samples(
            include_test_input
        )
        if include_test_input:
            test_samples = ['\t'.join(t) for t in test_samples]
        try:
            stream.write('\n'.join('{0}\t{1}'.format(gold, output)
                                   for gold, output in zip(
                                    test_samples, self.decoded)).encode('utf8'))
        except TypeError:
            stream.write('\n'.join('{0}\t{1}'.format(gold, output)
                                   for gold, output in zip(
                                    test_samples, self.decoded)))

    def decode_test(self, replace_pad=True):
        inv_vocab = {v: k for k, v in self.dataset.vocab_dec.items()}
        indices = np.zeros((self.test_out[0][0].shape[0],
                            len(self.test_out[0])))
        for si in range(len(self.test_out[0])):
            maxi = self.test_out[0][si].argmax(axis=1)
            indices[:, si] = maxi
        decoded = []
        for row in indices:
            dec = ''.join(inv_vocab[r] for r in row)
            if replace_pad:
                dec = dec.replace('PAD', ' ').strip()
                dec = dec.replace('STOP', ' ').strip()
            decoded.append(dec)
        self.decoded = decoded

    def populate_feed_dict(self, batch_enc, batch_dec):
        feed_dict = {}
        batch_size = batch_enc.shape[0]
        target_weights = np.ones((batch_size, batch_dec.shape[1]),
                                 dtype=np.float32)
        for i in range(batch_enc.shape[1]):
            feed_dict[self.enc_inp[i]] = batch_enc[:, i]
        for i in range(batch_dec.shape[1]):
            feed_dict[self.dec_inp[i]] = batch_dec[:, i]
            feed_dict[self.target_weights[i]] = target_weights[:, i]
        return feed_dict
<|endoftext|>


<commit_before>from fabric.api import local, task

@task
def preprocess_header():
    local('cpp -nostdinc spotify/api.h > spotify/api.processed.h || true')
<commit_msg>Add autotest task<commit_after>from fabric.api import execute, local, settings, task

@task
def preprocess_header():
    local('cpp -nostdinc spotify/api.h > spotify/api.processed.h || true')

@task
def test():
    local('nosetests')

@task
def autotest():
    while True:
        local('clear')
        with settings(warn_only=True):
            execute(test)
        local(
            'inotifywait -q -e create -e modify -e delete '
            '--exclude ".*\.(pyc|sw.)" -r spotify/ tests/')
<|endoftext|>