<commit_before>"""Contains the YamlInterface class for reading & writing YAML files.

Fixes for octal and boolean values are from here:
http://stackoverflow.com/questions/32965846/cant-parse-yaml-correctly/
"""
import copy
import logging
import re

import collections
import ruamel.yaml as yaml
from ruamel.yaml.reader import Reader
from ruamel.yaml.resolver import BaseResolver, Resolver
from ruamel.yaml.scanner import Scanner, RoundTripScanner
from ruamel.yaml.parser_ import Parser
from ruamel.yaml.composer import Composer
from ruamel.yaml.constructor import Constructor, RoundTripConstructor, ConstructorError
from ruamel.yaml.compat import to_str
from ruamel.yaml.dumper import RoundTripDumper

from mpf.core.file_manager import FileInterface, FileManager
from mpf.core.utility_functions import Util
from mpf._version import __version__, __config_version__

log = logging.getLogger('YAML Interface')

class MpfResolver(BaseResolver):

    """Resolver with mentioned fixes."""

    def __init__(self):
        """Initialise."""
        super().__init__()

MpfResolver.add_implicit_resolver(
    # Process any item beginning with a plus sign (+) as a string
    u'tag:yaml.org,2002:str',
    re.compile(
        u'''^(\\+([0-9a-zA-Z .]+))$''',
        re.X),
    list(u'+'))

MpfResolver.add_implicit_resolver(
    # Process any 3+ digit number with a leading zero as a string
    u'tag:yaml.org,2002:str',
    re.compile(
        u'''^(?:(0[0-9]{2,}))$''',
        re.X),
    list(u'0'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:bool',
    re.compile(
        u'''^(?:true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$''',
        re.X), list(u'tTfFyYnN'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:float',
    re.compile(u'''^(?:
     [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*
    |\\.[0-9_]+
    |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*
    |[-+]?\\.(?:inf|Inf|INF)
    |\\.(?:nan|NaN|NAN))$''', re.X),
    list(u'-+0123456789.'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:int',
    re.compile(u'''^(?:[-+]?0b[0-1_]+
    |[-+]?[0-9]+
    |[-+]?0o?[0-7_]+
    |[-+]?0x[0-9a-fA-F_]+
    |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
    list(u'-+0123456789'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:merge',
    re.compile(u'^(?:<<)$'),
    [u'<'])

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:null',
    re.compile(u'''^(?: ~
    |null|Null|NULL
    | )$''', re.X),
    [u'~', u'n', u'N', u''])

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:timestamp',
    re.compile(u'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
    |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
    (?:[Tt]|[ \\t]+)[0-9][0-9]?
    :[0-9][0-9] :[0-9][0-9] (?:\\.[0-9]*)?
    (?:[ \\t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
    list(u'0123456789'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:value',
    re.compile(u'^(?:=)$'),
    [u'='])

# The following resolver is only for documentation purposes. It cannot work
# because plain scalars cannot start with '!', '&', or '*'.
MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:yaml',
    re.compile(u'^(?:!|&|\\*)$'),
    list(u'!&*'))

class MpfConstructor(Constructor):

    """Constructor with fix."""

    def __init__(self):
        """Initialise."""
        super().__init__()

    def construct_mapping(self, node, deep=False):
        """Construct mapping but raise error when a section is defined twice.

        From: http://stackoverflow.com/questions/34358974/how-to-prevent-re-definition-of-keys-in-yaml
        """
        if not isinstance(node, yaml.MappingNode):
            raise ConstructorError(
                None, None,
                "expected a mapping node, but found %s" % node.id,
                node.start_mark)
        mapping = {}
        for key_node, value_node in node.value:
            # keys can be list -> deep
            key = self.construct_object(key_node, deep=True)
            # lists are not hashable, but tuples are
            if not isinstance(key, collections.Hashable):
                if isinstance(key, list):
                    key = tuple(key)
            if not isinstance(key, collections.Hashable):
                raise ConstructorError(
                    "while constructing a mapping", node.start_mark,
                    "found unhashable key", key_node.start_mark)

            value = self.construct_object(value_node, deep=deep)
            # next two lines differ from original
            if key in mapping:
                raise KeyError("Key \"{}\" was defined multiple times in config.".format(key))
            mapping[key] = value
        return mapping

class MpfRoundTripLoader(Reader, RoundTripScanner, Parser, Composer, RoundTripConstructor, MpfResolver):

    """Config loader which can roundtrip."""

    def __init__(self, stream):
        """Initialise loader."""
        Reader.__init__(self, stream)
        RoundTripScanner.__init__(self)
        Parser.__init__(self)
        Composer.__init__(self)
        RoundTripConstructor.__init__(self)
        MpfResolver.__init__(self)

class MpfLoader(Reader, Scanner, Parser, Composer, MpfConstructor, MpfResolver):

    """Config loader."""

    def __init__(self, stream):
        """Initialise loader."""
        Reader.__init__(self, stream)
        Scanner.__init__(self)
        Parser.__init__(self)
        Composer.__init__(self)
        MpfConstructor.__init__(self)
        MpfResolver.__init__(self)

for ch in list(u'yYnNoO'):
    del Resolver.yaml_implicit_resolvers[ch]

class YamlInterface(FileInterface):

    """File interface for yaml files."""

    file_types = ['.yaml', '.yml']
    cache = False
    file_cache = dict()

    @staticmethod
    def get_config_file_version(filename):
        """Return config file version."""
        with open(filename) as f:
            file_version = f.readline().split('config_version=')[-1:][0]

        try:
            return int(file_version)
        except ValueError:
            return 0

    @staticmethod
    def get_show_file_version(filename):
        """Return show file version."""
        with open(filename) as f:
            try:
                file_version = f.readline().split('show_version=')[-1:][0]
            except ValueError:
                return 0
        try:
            return int(file_version)
        except ValueError:
            return 0

    @staticmethod
    def check_config_file_version(filename):
        """Check to see if the version of the file name passed matches the config version MPF needs.

        Args:
            filename: The file with path to check.

        Raises:
            exception if the version of the file doesn't match what MPF needs.
        """
        filename = FileManager.locate_file(filename)
        file_interface = FileManager.get_file_interface(filename)
        file_version = file_interface.get_config_file_version(filename)

        if file_version != int(__config_version__):
            log.error("Config file %s is version %s. MPF %s requires "
                      "version %s", filename, file_version,
                      __version__, __config_version__)
            log.error("Use the Config File Migrator to automatically "
                      "migrate your config file to the latest version.")
            log.error("Migration tool: https://missionpinball.com/docs/tools/config-file-migrator/")
            log.error("More info on config version %s: "
                      "http://docs.missionpinball.org/docs/configuration-file"
                      "-reference/important-config-file-concepts/config_version/config-version-%s/",
                      __config_version__, __config_version__)
            return False
        else:
            return True

    def load(self, filename, verify_version=True, halt_on_error=True,
             round_trip=False):
        """Load a YAML file from disk.

        Args:
            filename: The file to load.
            verify_version: Boolean which specifies whether this method should
                verify whether this file's config_version is compatible with
                this version of MPF. Default is True.
            halt_on_error: Boolean which controls what happens if the file
                can't be loaded. (Not found, invalid format, etc. If True, MPF
                will raise an error and exit. If False, an empty config
                dictionary will be returned.
            round_trip: Boolean with controls if the round trip loader is used
                or if the regular MPFLoader will be used.  Default is False.

        Returns:
            A dictionary of the settings from this YAML file.
        """
        if YamlInterface.cache and filename in YamlInterface.file_cache:
            return copy.deepcopy(YamlInterface.file_cache[filename])

        if verify_version and not YamlInterface.check_config_file_version(filename):
            raise ValueError("Config file version mismatch: {}".format(filename))

        config = dict()

        try:
            self.log.debug("Loading file: %s", filename)

            with open(filename) as f:
                config = YamlInterface.process(f, round_trip)
        except yaml.YAMLError as exc:
            if hasattr(exc, 'problem_mark'):
                mark = exc.problem_mark
                self.log.debug("YAML error found in file %s. Line %s, "
                               "Position %s", filename, mark.line + 1,
                               mark.column + 1)
                if halt_on_error:
                    raise ValueError("YAML error found in file {}. Line {}, "
                                     "Position {}".format(filename, mark.line + 1, mark.column + 1))

            elif halt_on_error:
                raise ValueError("Error found in file %s" % filename)

        except Exception:   # pylint: disable-msg=broad-except
            self.log.debug("Couldn't load from file: %s", filename)

            if halt_on_error:
                raise ValueError("Error found in file %s" % filename)

        if YamlInterface.cache and config:
            YamlInterface.file_cache[filename] = copy.deepcopy(config)

        return config

    @staticmethod
    def process(data_string, round_trip=False):
        """Parse yaml from a string."""
        if round_trip:
            return yaml.load(data_string, Loader=MpfRoundTripLoader)
        else:
            return Util.keys_to_lower(yaml.load(data_string, Loader=MpfLoader))

    def save(self, filename, data, **kwargs):
        """Save config to yaml file."""
        try:
            include_comments = kwargs.pop('include_comments')
        except KeyError:
            include_comments = False

        with open(filename, 'w') as output_file:

            if include_comments:
                output_file.write(yaml.dump(data, Dumper=RoundTripDumper,
                                            **kwargs))
            else:
                output_file.write(yaml.dump(data, default_flow_style=False,
                                            **kwargs))

    @staticmethod
    def save_to_str(data):
        """Return yaml string from config."""
        return yaml.dump(data, Dumper=RoundTripDumper,
                         default_flow_style=False, indent=4, width=10)

    @staticmethod
    def rename_key(old_key, new_key, commented_map, logger=None):
        """Used to rename a key in YAML file data that was loaded with the RoundTripLoader (e.g. that contains comments.

        Comments are retained for the renamed key. Order of keys is also maintained.

        Args:
            old_key: The existing key name you want to change.
            new_key: The new key name.
            commented_map: The YAML data CommentMap class (from yaml.load) with
                the key you want to change.
            logger: Optional logger instance which will be used to log this at
                the debug level.

        Returns:
            The updated CommentedMap YAML dict. (Note that this method does not
            change the dict object (e.g. it's changed in place), you you most
            likely don't need to do anything with the returned dict.
        """
        if old_key == new_key or old_key not in commented_map:
            return commented_map

        key_list = list(commented_map.keys())

        for key in key_list:
            if key == old_key:

                if logger:
                    logger.debug('Renaming key: %s: -> %s:', old_key, new_key)

                commented_map[new_key] = commented_map[old_key]

                try:  # if there's no comment, it will not be in ca.items
                    commented_map.ca.items[new_key] = (
                        commented_map.ca.items.pop(old_key))
                except KeyError:
                    pass

                del commented_map[old_key]

            else:
                commented_map.move_to_end(key)

        return commented_map

    # pylint: disable-msg=too-many-arguments
    @staticmethod
    def copy_with_comments(source_dict, source_key, dest_dict, dest_key, delete_source=False, logger=None):
        """Copy dict with comments in config."""
        dest_dict[dest_key] = source_dict[source_key]

        try:
            dest_dict.ca.items[dest_key] = source_dict.ca.items[source_key]

            if logger and not delete_source:
                logger.debug('Copying key: %s -> %s', source_key, dest_key)

        except (KeyError, AttributeError):
            pass

        if delete_source:

            if logger:
                logger.debug('Moving key: %s -> %s', source_key, dest_key)

            del source_dict[source_key]
            source_dict.ca.items.pop(source_key, None)

    @staticmethod
    def del_key_with_comments(dic, key, logger=None):
        """Delete section with comments."""
        if key not in dic:
            return

        if logger:
            logger.debug("Removing key: %s", key)

        del dic[key]
        dic.ca.items.pop(key, None)

    @staticmethod
    def pretty_format(dic):
        """Return pretty printed config."""
        return '\r' + yaml.dump(dic, Dumper=RoundTripDumper, indent=4)

file_interface_class = YamlInterface
<commit_msg>remove nan and inf<commit_after>"""Contains the YamlInterface class for reading & writing YAML files.

Fixes for octal and boolean values are from here:
http://stackoverflow.com/questions/32965846/cant-parse-yaml-correctly/
"""
import copy
import logging
import re

import collections
import ruamel.yaml as yaml
from ruamel.yaml.reader import Reader
from ruamel.yaml.resolver import BaseResolver, Resolver
from ruamel.yaml.scanner import Scanner, RoundTripScanner
from ruamel.yaml.parser_ import Parser
from ruamel.yaml.composer import Composer
from ruamel.yaml.constructor import Constructor, RoundTripConstructor, ConstructorError
from ruamel.yaml.compat import to_str
from ruamel.yaml.dumper import RoundTripDumper

from mpf.core.file_manager import FileInterface, FileManager
from mpf.core.utility_functions import Util
from mpf._version import __version__, __config_version__

log = logging.getLogger('YAML Interface')

class MpfResolver(BaseResolver):

    """Resolver with mentioned fixes."""

    def __init__(self):
        """Initialise."""
        super().__init__()

MpfResolver.add_implicit_resolver(
    # Process any item beginning with a plus sign (+) as a string
    u'tag:yaml.org,2002:str',
    re.compile(
        u'''^(\\+([0-9a-zA-Z .]+))$''',
        re.X),
    list(u'+'))

MpfResolver.add_implicit_resolver(
    # Process any 3+ digit number with a leading zero as a string
    u'tag:yaml.org,2002:str',
    re.compile(
        u'''^(?:(0[0-9]{2,}))$''',
        re.X),
    list(u'0'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:bool',
    re.compile(
        u'''^(?:true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$''',
        re.X), list(u'tTfFyYnN'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:float',
    re.compile(u'''^(?:
     [-+]?(?:[0-9][0-9_]*)\\.[0-9_]*
    |\\.[0-9_]+
    |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]*)$''', re.X),
    list(u'-+0123456789.'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:int',
    re.compile(u'''^(?:[-+]?0b[0-1_]+
    |[-+]?[0-9]+
    |[-+]?0o?[0-7_]+
    |[-+]?0x[0-9a-fA-F_]+
    |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X),
    list(u'-+0123456789'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:merge',
    re.compile(u'^(?:<<)$'),
    [u'<'])

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:null',
    re.compile(u'''^(?: ~
    |null|Null|NULL
    | )$''', re.X),
    [u'~', u'n', u'N', u''])

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:timestamp',
    re.compile(u'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
    |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
    (?:[Tt]|[ \\t]+)[0-9][0-9]?
    :[0-9][0-9] :[0-9][0-9] (?:\\.[0-9]*)?
    (?:[ \\t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
    list(u'0123456789'))

MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:value',
    re.compile(u'^(?:=)$'),
    [u'='])

# The following resolver is only for documentation purposes. It cannot work
# because plain scalars cannot start with '!', '&', or '*'.
MpfResolver.add_implicit_resolver(
    u'tag:yaml.org,2002:yaml',
    re.compile(u'^(?:!|&|\\*)$'),
    list(u'!&*'))

class MpfConstructor(Constructor):

    """Constructor with fix."""

    def __init__(self):
        """Initialise."""
        super().__init__()

    def construct_mapping(self, node, deep=False):
        """Construct mapping but raise error when a section is defined twice.

        From: http://stackoverflow.com/questions/34358974/how-to-prevent-re-definition-of-keys-in-yaml
        """
        if not isinstance(node, yaml.MappingNode):
            raise ConstructorError(
                None, None,
                "expected a mapping node, but found %s" % node.id,
                node.start_mark)
        mapping = {}
        for key_node, value_node in node.value:
            # keys can be list -> deep
            key = self.construct_object(key_node, deep=True)
            # lists are not hashable, but tuples are
            if not isinstance(key, collections.Hashable):
                if isinstance(key, list):
                    key = tuple(key)
            if not isinstance(key, collections.Hashable):
                raise ConstructorError(
                    "while constructing a mapping", node.start_mark,
                    "found unhashable key", key_node.start_mark)

            value = self.construct_object(value_node, deep=deep)
            # next two lines differ from original
            if key in mapping:
                raise KeyError("Key \"{}\" was defined multiple times in config.".format(key))
            mapping[key] = value
        return mapping

class MpfRoundTripLoader(Reader, RoundTripScanner, Parser, Composer, RoundTripConstructor, MpfResolver):

    """Config loader which can roundtrip."""

    def __init__(self, stream):
        """Initialise loader."""
        Reader.__init__(self, stream)
        RoundTripScanner.__init__(self)
        Parser.__init__(self)
        Composer.__init__(self)
        RoundTripConstructor.__init__(self)
        MpfResolver.__init__(self)

class MpfLoader(Reader, Scanner, Parser, Composer, MpfConstructor, MpfResolver):

    """Config loader."""

    def __init__(self, stream):
        """Initialise loader."""
        Reader.__init__(self, stream)
        Scanner.__init__(self)
        Parser.__init__(self)
        Composer.__init__(self)
        MpfConstructor.__init__(self)
        MpfResolver.__init__(self)

for ch in list(u'yYnNoO'):
    del Resolver.yaml_implicit_resolvers[ch]

class YamlInterface(FileInterface):

    """File interface for yaml files."""

    file_types = ['.yaml', '.yml']
    cache = False
    file_cache = dict()

    @staticmethod
    def get_config_file_version(filename):
        """Return config file version."""
        with open(filename) as f:
            file_version = f.readline().split('config_version=')[-1:][0]

        try:
            return int(file_version)
        except ValueError:
            return 0

    @staticmethod
    def get_show_file_version(filename):
        """Return show file version."""
        with open(filename) as f:
            try:
                file_version = f.readline().split('show_version=')[-1:][0]
            except ValueError:
                return 0
        try:
            return int(file_version)
        except ValueError:
            return 0

    @staticmethod
    def check_config_file_version(filename):
        """Check to see if the version of the file name passed matches the config version MPF needs.

        Args:
            filename: The file with path to check.

        Raises:
            exception if the version of the file doesn't match what MPF needs.
        """
        filename = FileManager.locate_file(filename)
        file_interface = FileManager.get_file_interface(filename)
        file_version = file_interface.get_config_file_version(filename)

        if file_version != int(__config_version__):
            log.error("Config file %s is version %s. MPF %s requires "
                      "version %s", filename, file_version,
                      __version__, __config_version__)
            log.error("Use the Config File Migrator to automatically "
                      "migrate your config file to the latest version.")
            log.error("Migration tool: https://missionpinball.com/docs/tools/config-file-migrator/")
            log.error("More info on config version %s: "
                      "http://docs.missionpinball.org/docs/configuration-file"
                      "-reference/important-config-file-concepts/config_version/config-version-%s/",
                      __config_version__, __config_version__)
            return False
        else:
            return True

    def load(self, filename, verify_version=True, halt_on_error=True,
             round_trip=False):
        """Load a YAML file from disk.

        Args:
            filename: The file to load.
            verify_version: Boolean which specifies whether this method should
                verify whether this file's config_version is compatible with
                this version of MPF. Default is True.
            halt_on_error: Boolean which controls what happens if the file
                can't be loaded. (Not found, invalid format, etc. If True, MPF
                will raise an error and exit. If False, an empty config
                dictionary will be returned.
            round_trip: Boolean with controls if the round trip loader is used
                or if the regular MPFLoader will be used.  Default is False.

        Returns:
            A dictionary of the settings from this YAML file.
        """
        if YamlInterface.cache and filename in YamlInterface.file_cache:
            return copy.deepcopy(YamlInterface.file_cache[filename])

        if verify_version and not YamlInterface.check_config_file_version(filename):
            raise ValueError("Config file version mismatch: {}".format(filename))

        config = dict()

        try:
            self.log.debug("Loading file: %s", filename)

            with open(filename) as f:
                config = YamlInterface.process(f, round_trip)
        except yaml.YAMLError as exc:
            if hasattr(exc, 'problem_mark'):
                mark = exc.problem_mark
                self.log.debug("YAML error found in file %s. Line %s, "
                               "Position %s", filename, mark.line + 1,
                               mark.column + 1)
                if halt_on_error:
                    raise ValueError("YAML error found in file {}. Line {}, "
                                     "Position {}".format(filename, mark.line + 1, mark.column + 1))

            elif halt_on_error:
                raise ValueError("Error found in file %s" % filename)

        except Exception:   # pylint: disable-msg=broad-except
            self.log.debug("Couldn't load from file: %s", filename)

            if halt_on_error:
                raise ValueError("Error found in file %s" % filename)

        if YamlInterface.cache and config:
            YamlInterface.file_cache[filename] = copy.deepcopy(config)

        return config

    @staticmethod
    def process(data_string, round_trip=False):
        """Parse yaml from a string."""
        if round_trip:
            return yaml.load(data_string, Loader=MpfRoundTripLoader)
        else:
            return Util.keys_to_lower(yaml.load(data_string, Loader=MpfLoader))

    def save(self, filename, data, **kwargs):
        """Save config to yaml file."""
        try:
            include_comments = kwargs.pop('include_comments')
        except KeyError:
            include_comments = False

        with open(filename, 'w') as output_file:

            if include_comments:
                output_file.write(yaml.dump(data, Dumper=RoundTripDumper,
                                            **kwargs))
            else:
                output_file.write(yaml.dump(data, default_flow_style=False,
                                            **kwargs))

    @staticmethod
    def save_to_str(data):
        """Return yaml string from config."""
        return yaml.dump(data, Dumper=RoundTripDumper,
                         default_flow_style=False, indent=4, width=10)

    @staticmethod
    def rename_key(old_key, new_key, commented_map, logger=None):
        """Used to rename a key in YAML file data that was loaded with the RoundTripLoader (e.g. that contains comments.

        Comments are retained for the renamed key. Order of keys is also maintained.

        Args:
            old_key: The existing key name you want to change.
            new_key: The new key name.
            commented_map: The YAML data CommentMap class (from yaml.load) with
                the key you want to change.
            logger: Optional logger instance which will be used to log this at
                the debug level.

        Returns:
            The updated CommentedMap YAML dict. (Note that this method does not
            change the dict object (e.g. it's changed in place), you you most
            likely don't need to do anything with the returned dict.
        """
        if old_key == new_key or old_key not in commented_map:
            return commented_map

        key_list = list(commented_map.keys())

        for key in key_list:
            if key == old_key:

                if logger:
                    logger.debug('Renaming key: %s: -> %s:', old_key, new_key)

                commented_map[new_key] = commented_map[old_key]

                try:  # if there's no comment, it will not be in ca.items
                    commented_map.ca.items[new_key] = (
                        commented_map.ca.items.pop(old_key))
                except KeyError:
                    pass

                del commented_map[old_key]

            else:
                commented_map.move_to_end(key)

        return commented_map

    # pylint: disable-msg=too-many-arguments
    @staticmethod
    def copy_with_comments(source_dict, source_key, dest_dict, dest_key, delete_source=False, logger=None):
        """Copy dict with comments in config."""
        dest_dict[dest_key] = source_dict[source_key]

        try:
            dest_dict.ca.items[dest_key] = source_dict.ca.items[source_key]

            if logger and not delete_source:
                logger.debug('Copying key: %s -> %s', source_key, dest_key)

        except (KeyError, AttributeError):
            pass

        if delete_source:

            if logger:
                logger.debug('Moving key: %s -> %s', source_key, dest_key)

            del source_dict[source_key]
            source_dict.ca.items.pop(source_key, None)

    @staticmethod
    def del_key_with_comments(dic, key, logger=None):
        """Delete section with comments."""
        if key not in dic:
            return

        if logger:
            logger.debug("Removing key: %s", key)

        del dic[key]
        dic.ca.items.pop(key, None)

    @staticmethod
    def pretty_format(dic):
        """Return pretty printed config."""
        return '\r' + yaml.dump(dic, Dumper=RoundTripDumper, indent=4)

file_interface_class = YamlInterface
<|endoftext|>


<commit_before>"""Contains template code for a new hardware platform interface for MPF. You
can use this as the starting point for your own platform.

Note that if you create your own platform interface, we will be happy to add it
to the MPF package. That way we can help maintain it moving forward.

You can search-and-replace the word "template" (not case sensitive) with the
name of your own platform.

"""

import logging
from mpf.core.platform import Platform
from mpf.core.utility_functions import Util
from mpf.platform.interfaces.rgb_led_platform_interface import RGBLEDPlatformInterface
from mpf.platform.interfaces.matrix_light_platform_interface import MatrixLightPlatformInterface
from mpf.platform.interfaces.gi_platform_interface import GIPlatformInterface
from mpf.platform.interfaces.driver_platform_interface import DriverPlatformInterface

# you might have to add additional imports here for modules you need for your
# platform

class HardwarePlatform(Platform):
    """This is the base class for your hardware platform. Note that at this
    time, this class *must* be called HardwarePlatform."""

    def __init__(self, machine):
        super().__init__(machine)
        self.log = logging.getLogger("Template Platform")
        self.log.debug("Configuring template hardware interface.")

        # The following "features" are supposed to be constants that you can
        # use to define was is or is not in your own platform. However they
        # have not been implemented at this time. So for now just keep them
        # as-is below and we'll deal with them in some future version of MPF.
        self.features['max_pulse'] = 255
        self.features['hw_rule_coil_delay'] = False
        self.features['variable_recycle_time'] = False
        self.features['variable_debounce_time'] = False

        # Make the platform features available to everyone
        self.machine.config['platform'] = self.features

    def __repr__(self):
        """String name you'd like to show up in logs and stuff when a
        reference to this platform is printed."""
        return '<Platform.Template>'

    def configure_driver(self, config, device_type='coil'):
        """This method is called once per driver when MPF is starting up. It
        passes the config for the driver and returns a hardware driver object
        that can be used to control the driver.

        Args:
            config: A dict of configuration key/value settings that will be
                passed on to your driver class to create an instance of your
                driver. These can be whatever you want--whatever you need to
                setup a driver.
            device_type: String name of the type of MPF device that's being
                configured. This is needed since some platforms (like Williams
                WPC) use "drivers" to control coils, lights, GI, flashers, etc.
                Whether you need to act on this is up to you. Just know that
                when MPF calls this method, it will pass the config dict for
                this device as well as a string name of what type of device
                it's trying to setup.

        Returns:
            driver object, config number. The driver object that is returned
            is a your hardware driver that should have driver-like methods like
            pulse(), enable(), disable(), etc. It should already be mapped to
            the proper driver so it can be called directly like driver.pulse().
            Different types of drivers (e.g. coils vs. flashers) will have
            different methods available. (Keep reading for details.)

            The config number that's returned isn't really used. It's just
            stored by MPF in case this driver needs to be referenced by number
            in the future.

        """

        # In this example, we're passing the complete config dictionary to the
        # TemplateDriver constructor so it can set itself up as needed. You
        # might choose to only pass certain k/v pairs, or whatever else you
        # want.
        driver = TemplateDriver(config['number'])

        driver.driver_settings = config
        driver.driver_settings['pulse_ms'] = 30

        return driver, config['number']

    def configure_switch(self, config):
        """Called once per switch when MPF boots. It's used to setup the
        hardware switch interface.

        The main MPF loop will expect to receive notifications of switch
        changes in the main platform.tick() events. It will not poll every
        single switch. (If you need that behavior, you need to write it in the
        tick() method.

        Note: This method is similar to the configure_driver() method.

        Args:
            config: A dict of configuration key/value settings that will be
                passed on to your switch class to create an instance of your
                switch. These can be whatever you want--whatever you need to
                setup a switch.

        Returns:
            A reference to the instance of your harware switch class. This will
            be queried later to read the state of the switch.

        """

        # We want to have the virtual platform set all the initial switch states
        # to inactive, so we have to check the config.

        state = 0

        if config['type'].upper() == 'NC':
            state = 1

        self.hw_switches[config['number']] = state

        switch = VirtualSwitch(config['number'])

        switch.driver_settings = config

        return switch, config['number']

    def get_hw_switch_states(self):

        if not self.initial_states_sent:

            if 'virtual_platform_start_active_switches' in self.machine.config:

                initial_active_switches = [self.machine.switches[x].number for
                                           x in
                                           Util.string_to_list(
                                                   self.machine.config[
                                                       'virtual_platform_start_active_switches'])]

                for k, v in self.hw_switches.items():
                    if k in initial_active_switches:
                        self.hw_switches[k] ^= 1

            self.initial_states_sent = True

        else:
            switches = [x for x in self.machine.switches if x.platform == self]

            for switch in switches:
                self.hw_switches[switch.number] = switch.state ^ switch.invert

        return self.hw_switches

    def configure_accelerometer(self, device, number, useHighPass):
        pass

    def configure_matrixlight(self, config):
        return VirtualMatrixLight(config['number']), config['number']

    def configure_led(self, config):
        return VirtualLED(config['number'])

    def configure_gi(self, config):
        return VirtualGI(config['number']), config['number']

    def configure_dmd(self):
        return VirtualDMD(self.machine)

    def write_hw_rule(self, *args, **kwargs):
        pass

    def clear_hw_rule(self, sw_name):
        sw_num = self.machine.switches[sw_name].number

        for entry in list(self.hw_switch_rules.keys()):  # slice for copy
            if entry.startswith(
                    self.machine.switches.number(sw_num).name):
                del self.hw_switch_rules[entry]

    def i2c_write8(self, address, register, value):
        pass

    def i2c_read8(self, address, register):
        return None

    def i2c_read16(self, address, register):
        return None

class VirtualSwitch(object):
    """Represents a switch in a pinball machine used with virtual hardware."""

    def __init__(self, number):
        self.log = logging.getLogger('VirtualSwitch')
        self.number = number

class VirtualMatrixLight(MatrixLightPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualMatrixLight')
        self.number = number

    def on(self, brightness=255, fade_ms=0, start=0):
        pass

    def off(self):
        pass

class VirtualLED(RGBLEDPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualLED')
        self.number = number

    def color(self, color):
        # self.log.debug("Setting color: %s, fade: %s, comp: %s",
        #               color, fade_ms, brightness_compensation)
        pass

    def disable(self):
        pass

    def enable(self, brightness_compensation=True):
        pass

class VirtualGI(GIPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualGI')
        self.number = number

    def on(self, brightness):
        pass

    def off(self):
        pass

class TemplateDriver(DriverPlatformInterface):
    """The base class for a hardware driver on your platform. Several methods
    are required, including:

    disable()
    enable()
    pulse()

    The following are optional:
    state()
    tick()
    reconfigure()

    """

    def __init__(self, number):
        self.log = logging.getLogger('VirtualDriver')
        self.number = number

    def __repr__(self):
        return "VirtualDriver.{}".format(self.number)

    def disable(self):
        pass

    def enable(self):
        """Enables this driver, which means it's held "on" indefinitely until
        it's explicitly disabled.

        """
        # The actual code here will be the call to your library or whatever
        # that physically enables this driver. Remember this object is just for
        # this single driver, so it's up to you to set or save whatever you
        # nned in __init__() so that you know which driver to call from this
        # instance of the class.

        # for example (pseudocode):

        # self.serial_connection.send(self.number, 1)

        # or

        # self.my_driver(self.driver_number, command='enable', time=-1)

        pass

    def pulse(self, milliseconds=None):
        """Pulses this driver for a pre-determined amount of time, after which
        this driver is turned off automatically. Note that on most platforms,
        pulse times are a max of 255ms. (Beyond that MPF will send separate
        enable() and disable() commands.

        Args:
            milliseconds: The number of ms to pulse this driver for. You should
                raise a ValueError if the value is out of range for your
                platform. If this value is None, you should pulse this driver
                with a default setting. You can set the default in the driver
                config via your __init__() method, or you can pick some default
                that's hard coded.

        Returns:
            A integer of the actual time this driver is going to be pulsed for.
            MPF uses this for timing in certain situations to make sure too
            many drivers aren't activated at once.

        """
        if not milliseconds:
            milliseconds = self.driver_settings['pulse_ms']

        # do the actual hardware pulse... whatever that looks like for your
        # platform

        return milliseconds

    def state(self):
        pass

    def tick(self):
        pass

    def reconfigure(self, polarity):
        pass

class VirtualDMD(object):
    def __init__(self, machine):
        pass

    def update(self, data):
        pass
<commit_msg>initialise variable before use<commit_after>"""Contains template code for a new hardware platform interface for MPF. You
can use this as the starting point for your own platform.

Note that if you create your own platform interface, we will be happy to add it
to the MPF package. That way we can help maintain it moving forward.

You can search-and-replace the word "template" (not case sensitive) with the
name of your own platform.

"""

import logging
from mpf.core.platform import Platform
from mpf.core.utility_functions import Util
from mpf.platform.interfaces.rgb_led_platform_interface import RGBLEDPlatformInterface
from mpf.platform.interfaces.matrix_light_platform_interface import MatrixLightPlatformInterface
from mpf.platform.interfaces.gi_platform_interface import GIPlatformInterface
from mpf.platform.interfaces.driver_platform_interface import DriverPlatformInterface

# you might have to add additional imports here for modules you need for your
# platform

class HardwarePlatform(Platform):
    """This is the base class for your hardware platform. Note that at this
    time, this class *must* be called HardwarePlatform."""

    def __init__(self, machine):
        super().__init__(machine)
        self.log = logging.getLogger("Template Platform")
        self.log.debug("Configuring template hardware interface.")
        self.initial_states_sent = False

        # The following "features" are supposed to be constants that you can
        # use to define was is or is not in your own platform. However they
        # have not been implemented at this time. So for now just keep them
        # as-is below and we'll deal with them in some future version of MPF.
        self.features['max_pulse'] = 255
        self.features['hw_rule_coil_delay'] = False
        self.features['variable_recycle_time'] = False
        self.features['variable_debounce_time'] = False

        # Make the platform features available to everyone
        self.machine.config['platform'] = self.features

    def __repr__(self):
        """String name you'd like to show up in logs and stuff when a
        reference to this platform is printed."""
        return '<Platform.Template>'

    def configure_driver(self, config, device_type='coil'):
        """This method is called once per driver when MPF is starting up. It
        passes the config for the driver and returns a hardware driver object
        that can be used to control the driver.

        Args:
            config: A dict of configuration key/value settings that will be
                passed on to your driver class to create an instance of your
                driver. These can be whatever you want--whatever you need to
                setup a driver.
            device_type: String name of the type of MPF device that's being
                configured. This is needed since some platforms (like Williams
                WPC) use "drivers" to control coils, lights, GI, flashers, etc.
                Whether you need to act on this is up to you. Just know that
                when MPF calls this method, it will pass the config dict for
                this device as well as a string name of what type of device
                it's trying to setup.

        Returns:
            driver object, config number. The driver object that is returned
            is a your hardware driver that should have driver-like methods like
            pulse(), enable(), disable(), etc. It should already be mapped to
            the proper driver so it can be called directly like driver.pulse().
            Different types of drivers (e.g. coils vs. flashers) will have
            different methods available. (Keep reading for details.)

            The config number that's returned isn't really used. It's just
            stored by MPF in case this driver needs to be referenced by number
            in the future.

        """

        # In this example, we're passing the complete config dictionary to the
        # TemplateDriver constructor so it can set itself up as needed. You
        # might choose to only pass certain k/v pairs, or whatever else you
        # want.
        driver = TemplateDriver(config['number'])

        driver.driver_settings = config
        driver.driver_settings['pulse_ms'] = 30

        return driver, config['number']

    def configure_switch(self, config):
        """Called once per switch when MPF boots. It's used to setup the
        hardware switch interface.

        The main MPF loop will expect to receive notifications of switch
        changes in the main platform.tick() events. It will not poll every
        single switch. (If you need that behavior, you need to write it in the
        tick() method.

        Note: This method is similar to the configure_driver() method.

        Args:
            config: A dict of configuration key/value settings that will be
                passed on to your switch class to create an instance of your
                switch. These can be whatever you want--whatever you need to
                setup a switch.

        Returns:
            A reference to the instance of your harware switch class. This will
            be queried later to read the state of the switch.

        """

        # We want to have the virtual platform set all the initial switch states
        # to inactive, so we have to check the config.

        state = 0

        if config['type'].upper() == 'NC':
            state = 1

        self.hw_switches[config['number']] = state

        switch = VirtualSwitch(config['number'])

        switch.driver_settings = config

        return switch, config['number']

    def get_hw_switch_states(self):

        if not self.initial_states_sent:

            if 'virtual_platform_start_active_switches' in self.machine.config:

                initial_active_switches = [self.machine.switches[x].number for
                                           x in
                                           Util.string_to_list(
                                                   self.machine.config[
                                                       'virtual_platform_start_active_switches'])]

                for k, v in self.hw_switches.items():
                    if k in initial_active_switches:
                        self.hw_switches[k] ^= 1

            self.initial_states_sent = True

        else:
            switches = [x for x in self.machine.switches if x.platform == self]

            for switch in switches:
                self.hw_switches[switch.number] = switch.state ^ switch.invert

        return self.hw_switches

    def configure_accelerometer(self, device, number, useHighPass):
        pass

    def configure_matrixlight(self, config):
        return VirtualMatrixLight(config['number']), config['number']

    def configure_led(self, config):
        return VirtualLED(config['number'])

    def configure_gi(self, config):
        return VirtualGI(config['number']), config['number']

    def configure_dmd(self):
        return VirtualDMD(self.machine)

    def write_hw_rule(self, *args, **kwargs):
        pass

    def clear_hw_rule(self, sw_name):
        sw_num = self.machine.switches[sw_name].number

        for entry in list(self.hw_switch_rules.keys()):  # slice for copy
            if entry.startswith(
                    self.machine.switches.number(sw_num).name):
                del self.hw_switch_rules[entry]

    def i2c_write8(self, address, register, value):
        pass

    def i2c_read8(self, address, register):
        return None

    def i2c_read16(self, address, register):
        return None

class VirtualSwitch(object):
    """Represents a switch in a pinball machine used with virtual hardware."""

    def __init__(self, number):
        self.log = logging.getLogger('VirtualSwitch')
        self.number = number

class VirtualMatrixLight(MatrixLightPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualMatrixLight')
        self.number = number

    def on(self, brightness=255, fade_ms=0, start=0):
        pass

    def off(self):
        pass

class VirtualLED(RGBLEDPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualLED')
        self.number = number

    def color(self, color):
        # self.log.debug("Setting color: %s, fade: %s, comp: %s",
        #               color, fade_ms, brightness_compensation)
        pass

    def disable(self):
        pass

    def enable(self, brightness_compensation=True):
        pass

class VirtualGI(GIPlatformInterface):
    def __init__(self, number):
        self.log = logging.getLogger('VirtualGI')
        self.number = number

    def on(self, brightness):
        pass

    def off(self):
        pass

class TemplateDriver(DriverPlatformInterface):
    """The base class for a hardware driver on your platform. Several methods
    are required, including:

    disable()
    enable()
    pulse()

    The following are optional:
    state()
    tick()
    reconfigure()

    """

    def __init__(self, number):
        self.log = logging.getLogger('VirtualDriver')
        self.number = number

    def __repr__(self):
        return "VirtualDriver.{}".format(self.number)

    def disable(self):
        pass

    def enable(self):
        """Enables this driver, which means it's held "on" indefinitely until
        it's explicitly disabled.

        """
        # The actual code here will be the call to your library or whatever
        # that physically enables this driver. Remember this object is just for
        # this single driver, so it's up to you to set or save whatever you
        # nned in __init__() so that you know which driver to call from this
        # instance of the class.

        # for example (pseudocode):

        # self.serial_connection.send(self.number, 1)

        # or

        # self.my_driver(self.driver_number, command='enable', time=-1)

        pass

    def pulse(self, milliseconds=None):
        """Pulses this driver for a pre-determined amount of time, after which
        this driver is turned off automatically. Note that on most platforms,
        pulse times are a max of 255ms. (Beyond that MPF will send separate
        enable() and disable() commands.

        Args:
            milliseconds: The number of ms to pulse this driver for. You should
                raise a ValueError if the value is out of range for your
                platform. If this value is None, you should pulse this driver
                with a default setting. You can set the default in the driver
                config via your __init__() method, or you can pick some default
                that's hard coded.

        Returns:
            A integer of the actual time this driver is going to be pulsed for.
            MPF uses this for timing in certain situations to make sure too
            many drivers aren't activated at once.

        """
        if not milliseconds:
            milliseconds = self.driver_settings['pulse_ms']

        # do the actual hardware pulse... whatever that looks like for your
        # platform

        return milliseconds

    def state(self):
        pass

    def tick(self):
        pass

    def reconfigure(self, polarity):
        pass

class VirtualDMD(object):
    def __init__(self, machine):
        pass

    def update(self, data):
        pass
<|endoftext|>


<commit_before><commit_msg>fix migration<commit_after><|endoftext|>


<commit_before>#!/usr/bin/env python

# run the continuous integration process
# includes watching the git repo for pull requests and commits

import os
import sys
import requests
import json
from time import sleep, gmtime, strftime
from subprocess import call, check_output, CalledProcessError

repo_owner = 'snhack'
repo_name = 'LogoBot'

repos_rel_dir = '../../'
primary_repo_dir = 'LogoBot'
staging_repo_dir = primary_repo_dir + 'Staging'

ci_log_name = 'ci.log'
prhist = []

def dict_in_array(dict, key, value):
    res = False
    for item in dict:
        if item[key] == value:
            res = True
            exit
    return res

def poll(un, pw, proxies):

    cilog = open(ci_log_name, 'a+')
    cilog.seek(0)

    # load cilog into prhist
    lines = cilog.readlines()
    for line in lines:
        line = line.split('_')
        num = int(line[0])
        dt = line[1]
        if not dict_in_array(prhist, 'number', num):
            prhist.append({'number':num, 'updated_at':dt})

    print("Polling for pull requests for commits...")
    print("")

    while True:
        print(strftime("%H:%M:%S", gmtime()))
        print("Getting list of pull requests...")

        r = requests.get('https://api.github.com/repos/'+repo_owner+'/'+repo_name+'/pulls', auth=(un, pw), proxies=proxies)

        jso = r.json()

        print("  Found: "+str(len(jso))+" pull request(s)")

        for p in jso:
            print("Checking: #"+str(p['number']) + " - "+ p['title'] + " by "+p['user']['login'])
            """
            print(p['body'])
            print(p['state'])
            print(p['merged_at'])
            print(p['updated_at'])
            """

            # check if the pull request is ready to be merged
            if p['state'] == 'open' and p['merged_at']== None:

                # check if we've done it before?
                if not dict_in_array(prhist, 'number', p['number']):
                    try:
                        errorlevel = 0

                        oplog = ''

                        comments_url = p['_links']['comments']['href']

                        # comment
                        payload = {
                            'body':'CI: Starting build process...'
                        }
                        r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                        # Discard any local changes
                        print("  Clean working tree")
                        o = check_output(['git','checkout','--','.'])
                        print(o)
                        oplog += "\nClean working tree\n"
                        oplog += o

                        print("  Pull master")
                        o = check_output(['git','pull','origin','master'])
                        print(o)
                        oplog += "\nPull master\n"
                        oplog += o

                        branch = p['head']['ref']

                        print("  Checkout master")
                        o = check_output(['git','checkout','master'])
                        print(o)
                        oplog += "\nCheckout master\n"
                        oplog += o

                        print("  Merge branch: "+branch)
                        o = ''
                        try:
                            o = check_output(['git','merge','origin/'+branch])
                            print(o)
                        except CalledProcessError as e:
                            print("  Error: "+ str(e.returncode))
                            errorlevel = e.returncode

                        oplog += "\nMerge branch: origin/"+branch+"\n"
                        oplog += o

                        if errorlevel == 0:
                            # Now run the build process
                            print("  Building")

                            os.chdir('hardware/ci')
                            o = ''
                            try:
                                o = check_output(['./build.py'])
                                print(o)
                            except CalledProcessError as e:
                                print("  Error: "+ str(e.returncode))
                                errorlevel = 1

                            os.chdir('../../')

                            oplog += "\nBuilding\n"
                            oplog += o

                        if errorlevel == 0:
                            print("  Passed, auto-merging into master...")

                            # comment
                            payload = {
                                'body':'CI: Build process successful - auto-merging into master\n\nLog\n' + oplog
                            }
                            r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                            # merge
                            payload = {
                                'commit_message':p['title']
                            }
                            r = requests.post(p['_links']['self']['href'] + '/merge',
                                              auth=(un, pw), proxies=proxies, data=json.dumps(payload))
                            print(r)

                        else:
                            print("  Errors, adding to pull request comments...")

                            # log the error
                            payload = {
                                'body':'CI: Unable to auto-merge, build process encountered errors\n\nLog\n' + oplog
                            }
                            r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                        # Log this request so we don't process it again
                        hist = {'number':p['number'], 'updated_at':p['updated_at']}
                        prhist.append(hist)
                        cilog.write(str(p['number']) + '_' + p['updated_at'] + '_' + str(errorlevel)+'\n')
                        cilog.flush()

                        print("  Done")

                    except CalledProcessError as e:
                        print("Error: "+ str(e.returncode))
                else:
                    print("  Skipping")

            else:
                print("  Error: Pull request not open or already merged")

        print("")
        sleep(60)
        call(['clear'])

    cilog.close()

def ci(un, pw, http_proxy="", https_proxy=""):
    print("Continuous Integration")
    print("----------------------")

    proxies = {
      "http": http_proxy,
      "https": https_proxy,
    }

    print("")
    print("Checking connection to github...")

    r = requests.get('https://api.github.com/user', auth=(un, pw), proxies=proxies)
    if r.status_code == 200:
        print("  OK")

        print("Changing working directory...")
        os.chdir(repos_rel_dir)

        print("  Now in: "+os.getcwd())

        print("Changing to staging dir: "+staging_repo_dir)
        if os.path.isdir(staging_repo_dir):
            os.chdir(staging_repo_dir)
            print("  OK")

            # Could check for empty dir here and if so, do a git clone?
            # git clone git@github.com:snhack/LogoBot .

            contents = os.listdir('.')
            if len(contents) == 0:
                print("  Staging empty - cloning repo")
                o = check_output(['git','clone','git@github.com:'+repo_owner+'/'+repo_name,'.'])

            poll(un, pw, proxies)

        else:
            print("  Error: Staging dir does not exist")

    else:
        print("  Error")
        print("  Status Code: "+r.status_code)
        print("  Response: "+r.text)

    # o = check_output(['git','branch'])

if __name__ == '__main__':
    if len(sys.argv) == 3:
        ci(sys.argv[1], sys.argv[2], "", "")
    elif len(sys.argv) > 3:
        ci(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
    else:
        print("Usage: ci <git-username> <git-password> <http_proxy> <https_proxy>")<commit_msg>trying to get auto-merge to work properly<commit_after>#!/usr/bin/env python

# run the continuous integration process
# includes watching the git repo for pull requests and commits

import os
import sys
import requests
import json
from time import sleep, gmtime, strftime
from subprocess import call, check_output, CalledProcessError

repo_owner = 'snhack'
repo_name = 'LogoBot'

repos_rel_dir = '../../'
primary_repo_dir = 'LogoBot'
staging_repo_dir = primary_repo_dir + 'Staging'

ci_log_name = 'ci.log'
prhist = []

def dict_in_array(dict, key, value):
    res = False
    for item in dict:
        if item[key] == value:
            res = True
            exit
    return res

def poll(un, pw, proxies):

    cilog = open(ci_log_name, 'a+')
    cilog.seek(0)

    # load cilog into prhist
    lines = cilog.readlines()
    for line in lines:
        line = line.split('_')
        num = int(line[0])
        dt = line[1]
        if not dict_in_array(prhist, 'number', num):
            prhist.append({'number':num, 'updated_at':dt})

    print("Polling for pull requests for commits...")
    print("")

    while True:
        print(strftime("%H:%M:%S", gmtime()))
        print("Getting list of pull requests...")

        r = requests.get('https://api.github.com/repos/'+repo_owner+'/'+repo_name+'/pulls', auth=(un, pw), proxies=proxies)

        jso = r.json()

        print("  Found: "+str(len(jso))+" pull request(s)")

        for p in jso:
            print("Checking: #"+str(p['number']) + " - "+ p['title'] + " by "+p['user']['login'])
            """
            print(p['body'])
            print(p['state'])
            print(p['merged_at'])
            print(p['updated_at'])
            """

            # check if the pull request is ready to be merged
            if p['state'] == 'open' and p['merged_at']== None:

                # check if we've done it before?
                if not dict_in_array(prhist, 'number', p['number']):
                    try:
                        errorlevel = 0

                        oplog = ''

                        comments_url = p['_links']['comments']['href']

                        # comment
                        payload = {
                            'body':'CI: Starting build process...'
                        }
                        r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                        # Discard any local changes
                        print("  Clean working tree")
                        o = check_output(['git','checkout','--','.'])
                        print(o)
                        oplog += "\nClean working tree\n"
                        oplog += o

                        print("  Pull master")
                        o = check_output(['git','pull','origin','master'])
                        print(o)
                        oplog += "\nPull master\n"
                        oplog += o

                        branch = p['head']['ref']

                        print("  Checkout master")
                        o = check_output(['git','checkout','master'])
                        print(o)
                        oplog += "\nCheckout master\n"
                        oplog += o

                        print("  Merge branch: "+branch)
                        o = ''
                        try:
                            o = check_output(['git','merge','origin/'+branch])
                            print(o)
                        except CalledProcessError as e:
                            print("  Error: "+ str(e.returncode))
                            errorlevel = e.returncode

                        oplog += "\nMerge branch: origin/"+branch+"\n"
                        oplog += o

                        if errorlevel == 0:
                            # Now run the build process
                            print("  Building")

                            os.chdir('hardware/ci')
                            o = ''
                            try:
                                o = check_output(['./build.py'])
                                print(o)
                            except CalledProcessError as e:
                                print("  Error: "+ str(e.returncode))
                                errorlevel = 1

                            os.chdir('../../')

                            oplog += "\nBuilding\n"
                            oplog += o

                        if errorlevel == 0:
                            print("  Passed, auto-merging into master...")

                            # comment
                            payload = {
                                'body':'CI: Build process successful - auto-merging into master\n\nLog\n' + oplog
                            }
                            r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                            # merge
                            payload = {
                                'commit_message':p['title']
                            }
                            r = requests.put(p['_links']['self']['href'] + '/merge',
                                              auth=(un, pw), proxies=proxies, data=json.dumps(payload))
                            print(r)

                        else:
                            print("  Errors, adding to pull request comments...")

                            # log the error
                            payload = {
                                'body':'CI: Unable to auto-merge, build process encountered errors\n\nLog\n' + oplog
                            }
                            r = requests.post(comments_url, auth=(un, pw), proxies=proxies, data=json.dumps(payload))

                        # Log this request so we don't process it again
                        hist = {'number':p['number'], 'updated_at':p['updated_at']}
                        prhist.append(hist)
                        cilog.write(str(p['number']) + '_' + p['updated_at'] + '_' + str(errorlevel)+'\n')
                        cilog.flush()

                        print("  Done")

                    except CalledProcessError as e:
                        print("Error: "+ str(e.returncode))
                else:
                    print("  Skipping")

            else:
                print("  Error: Pull request not open or already merged")

        print("")
        sleep(60)
        call(['clear'])

    cilog.close()

def ci(un, pw, http_proxy="", https_proxy=""):
    print("Continuous Integration")
    print("----------------------")

    proxies = {
      "http": http_proxy,
      "https": https_proxy,
    }

    print("")
    print("Checking connection to github...")

    r = requests.get('https://api.github.com/user', auth=(un, pw), proxies=proxies)
    if r.status_code == 200:
        print("  OK")

        print("Changing working directory...")
        os.chdir(repos_rel_dir)

        print("  Now in: "+os.getcwd())

        print("Changing to staging dir: "+staging_repo_dir)
        if os.path.isdir(staging_repo_dir):
            os.chdir(staging_repo_dir)
            print("  OK")

            # Could check for empty dir here and if so, do a git clone?
            # git clone git@github.com:snhack/LogoBot .

            contents = os.listdir('.')
            if len(contents) == 0:
                print("  Staging empty - cloning repo")
                o = check_output(['git','clone','git@github.com:'+repo_owner+'/'+repo_name,'.'])

            poll(un, pw, proxies)

        else:
            print("  Error: Staging dir does not exist")

    else:
        print("  Error")
        print("  Status Code: "+r.status_code)
        print("  Response: "+r.text)

    # o = check_output(['git','branch'])

if __name__ == '__main__':
    if len(sys.argv) == 3:
        ci(sys.argv[1], sys.argv[2], "", "")
    elif len(sys.argv) > 3:
        ci(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])
    else:
        print("Usage: ci <git-username> <git-password> <http_proxy> <https_proxy>")<|endoftext|>


<commit_before><commit_msg>Sometimes couch_form.problem does not exist<commit_after><|endoftext|>


<commit_before>from __future__ import print_function

import os
import sys
import json
import time
import pytest

from nb_conda_kernels.discovery import CondaKernelProvider
from nb_conda_kernels.manager import RUNNER_COMMAND
from jupyter_client.blocking.client import Empty

START_TIMEOUT = 10
CMD_TIMEOUT = 1
NUM_RETRIES = 5
is_win = sys.platform.startswith('win')
is_py2 = sys.version_info[0] < 3

provider = CondaKernelProvider()

old_print = print
def print(x):
    old_print('\n'.join(json.dumps(y)[1:-1] for y in x.splitlines()))
    sys.stdout.flush()

if is_win:
    # Create a job object and assign ourselves to it, so that
    # all remaining test subprocesses get killed off on completion.
    # This prevents AppVeyor from waiting an hour
    # https://stackoverflow.com/a/23587108 (and its first comment)
    import win32api, win32con, win32job  # noqa
    hJob = win32job.CreateJobObject(None, "")
    extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation)
    extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
    win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
    perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
    hProcess = win32api.OpenProcess(perms, False, os.getpid())
    win32job.AssignProcessToJobObject(hJob, hProcess)

def find_test_keys():
    if os.environ.get('CONDA_BUILD'):
        # The current version of conda build manually adds the activation
        # directories to the PATH---and then calls the standard conda
        # activation script, which does it again. This frustrates conda's
        # ability to deactivate this environment. Most package builds are
        # not affected by this, but we are, because our tests need to do
        # environment activation and deactivation. To fix this, we remove
        # the duplicate PATH entries conda-build added.
        print('BEFORE: {}'.format(os.environ['PATH']))
        path_list = os.environ['PATH'].split(os.pathsep)
        path_dups = set()
        path_list = [p for p in path_list
                     if not p.startswith(sys.prefix) or
                     p not in path_dups and not path_dups.add(p)]
        os.environ['PATH'] = os.pathsep.join(path_list)
        print('AFTER: {}'.format(os.environ['PATH']))
    keys = []
    for key, _ in provider.find_kernels():
        assert key.startswith('conda-')
        if key.endswith('-py') or key.endswith('-r'):
            keys.append(key)
    return keys

@pytest.mark.parametrize("key", find_test_keys())
def test_runner(key):
    kernel_manager = provider.make_manager(key)
    if kernel_manager.kernel_spec.argv[:3] == RUNNER_COMMAND:
        env_path = kernel_manager.kernel_spec.argv[4]
    else:
        env_path = sys.prefix
    env_path_fs = env_path.replace('\\', '/')
    valid = False
    # For reasons we do not fully understand, the kernels sometimes die immediately
    # and sometimes hang in this loop. Frankly the purpose of this test is not to
    # understand why that is but to simply test that a successfully run kernel is
    # using the correct environment. So we're using a simple retry loop, and we
    # use a timeout when waiting for messages from the kernel.
    for tries in range(NUM_RETRIES):
        outputs = []
        client = None
        try:
            print('\n--- attempt {}'.format(tries+1))
            kernel_manager.start_kernel()
            client = kernel_manager.client()
            client.start_channels()
            client.wait_for_ready(timeout=START_TIMEOUT)
            if key.endswith('-r'):
                commands = ['cat(Sys.getenv("CONDA_PREFIX"),fill=TRUE)',
                            'cat(dirname(dirname(dirname(.libPaths()))),fill=TRUE)',
                            'quit(save="no")']
            else:
                commands = ['import os, sys',
                            'print(os.environ["CONDA_PREFIX"])',
                            'print(sys.prefix)',
                            'quit']
            for command in commands:
                print('>>> {}'.format(command))
                m_id = client.execute(command)
                while True:
                    msg = client.get_iopub_msg(timeout=CMD_TIMEOUT)['content']
                    if msg.get('execution_state') == 'idle':
                        break
                    if msg.get('name') == 'stdout':
                        outputs.append(msg['text'].strip())
                        print(outputs[-1])
            valid = True
        except:
            time.sleep(CMD_TIMEOUT)
            pass
        finally:
            if client is not None:
                client.stop_channels()
            if kernel_manager.is_alive():
                kernel_manager.request_shutdown()
                kernel_manager.finish_shutdown()
        if valid:
            break
    else:
        assert False, 'Did not successfully run kernel'
    assert valid and len(outputs) >= 2 and all(o in (env_path, env_path_fs) for o in outputs[-2:])

if __name__ == '__main__':
    for key in find_test_keys():
        test_runner(key)
<commit_msg>Fix case issues<commit_after>from __future__ import print_function

import os
import sys
import json
import time
import pytest

from nb_conda_kernels.discovery import CondaKernelProvider
from nb_conda_kernels.manager import RUNNER_COMMAND
from jupyter_client.blocking.client import Empty

START_TIMEOUT = 10
CMD_TIMEOUT = 1
NUM_RETRIES = 5
is_win = sys.platform.startswith('win')
is_py2 = sys.version_info[0] < 3

provider = CondaKernelProvider()

old_print = print
def print(x):
    old_print('\n'.join(json.dumps(y)[1:-1] for y in x.splitlines()))
    sys.stdout.flush()

if is_win:
    # Create a job object and assign ourselves to it, so that
    # all remaining test subprocesses get killed off on completion.
    # This prevents AppVeyor from waiting an hour
    # https://stackoverflow.com/a/23587108 (and its first comment)
    import win32api, win32con, win32job  # noqa
    hJob = win32job.CreateJobObject(None, "")
    extended_info = win32job.QueryInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation)
    extended_info['BasicLimitInformation']['LimitFlags'] = win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
    win32job.SetInformationJobObject(hJob, win32job.JobObjectExtendedLimitInformation, extended_info)
    perms = win32con.PROCESS_TERMINATE | win32con.PROCESS_SET_QUOTA
    hProcess = win32api.OpenProcess(perms, False, os.getpid())
    win32job.AssignProcessToJobObject(hJob, hProcess)

def find_test_keys():
    if os.environ.get('CONDA_BUILD'):
        # The current version of conda build manually adds the activation
        # directories to the PATH---and then calls the standard conda
        # activation script, which does it again. This frustrates conda's
        # ability to deactivate this environment. Most package builds are
        # not affected by this, but we are, because our tests need to do
        # environment activation and deactivation. To fix this, we remove
        # the duplicate PATH entries conda-build added.
        print('BEFORE: {}'.format(os.environ['PATH']))
        path_list = os.environ['PATH'].split(os.pathsep)
        path_dups = set()
        path_list = [p for p in path_list
                     if not p.startswith(sys.prefix) or
                     p not in path_dups and not path_dups.add(p)]
        os.environ['PATH'] = os.pathsep.join(path_list)
        print('AFTER: {}'.format(os.environ['PATH']))
    keys = []
    for key, _ in provider.find_kernels():
        assert key.startswith('conda-')
        if key.endswith('-py') or key.endswith('-r'):
            keys.append(key)
    return keys

@pytest.mark.parametrize("key", find_test_keys())
def test_runner(key):
    kernel_manager = provider.make_manager(key)
    if kernel_manager.kernel_spec.argv[:3] == RUNNER_COMMAND:
        env_path = kernel_manager.kernel_spec.argv[4]
    else:
        env_path = sys.prefix
    env_path = os.path.normcase(os.path.normpath(env_path))
    print(env_path)
    valid = False
    # For reasons we do not fully understand, the kernels sometimes die immediately
    # and sometimes hang in this loop. Frankly the purpose of this test is not to
    # understand why that is but to simply test that a successfully run kernel is
    # using the correct environment. So we're using a simple retry loop, and we
    # use a timeout when waiting for messages from the kernel.
    for tries in range(NUM_RETRIES):
        outputs = []
        client = None
        try:
            print('\n--- attempt {}'.format(tries+1))
            kernel_manager.start_kernel()
            client = kernel_manager.client()
            client.start_channels()
            client.wait_for_ready(timeout=START_TIMEOUT)
            if key.endswith('-r'):
                commands = ['cat(Sys.getenv("CONDA_PREFIX"),fill=TRUE)',
                            'cat(dirname(dirname(dirname(.libPaths()))),fill=TRUE)',
                            'quit(save="no")']
            else:
                commands = ['import os, sys',
                            'print(os.environ["CONDA_PREFIX"])',
                            'print(sys.prefix)',
                            'quit']
            for command in commands:
                print('>>> {}'.format(command))
                m_id = client.execute(command)
                while True:
                    msg = client.get_iopub_msg(timeout=CMD_TIMEOUT)['content']
                    if msg.get('execution_state') == 'idle':
                        break
                    if msg.get('name') == 'stdout':
                        outputs.append(msg['text'].strip())
                        print(outputs[-1])
            valid = True
        except:
            time.sleep(CMD_TIMEOUT)
            pass
        finally:
            if client is not None:
                client.stop_channels()
            if kernel_manager.is_alive():
                kernel_manager.request_shutdown()
                kernel_manager.finish_shutdown()
        if valid:
            break
    else:
        assert False, 'Did not successfully run kernel'
    assert valid and len(outputs) >= 2
    for o in outputs[-2:]:
        assert os.path.normcase(os.path.normpath(o)) == env_path, (o, env_path)

if __name__ == '__main__':
    for key in find_test_keys():
        test_runner(key)
<|endoftext|>